From ad9512a1fc1b16b9cc207bad7f24b3ee7fd273fb Mon Sep 17 00:00:00 2001 From: Mikhail Leonov Date: Tue, 4 Aug 2026 12:04:43 +0600 Subject: [PATCH 1/6] feat: enhance speech handling by removing fenced code blocks --- .../src/remove-fenced-code-blocks.ts | 145 ++++++++++++++++++ .../speech-streaming-service/src/service.ts | 5 +- 2 files changed, 149 insertions(+), 1 deletion(-) create mode 100644 libs/mobile/shared/data-access/speech-streaming-service/src/remove-fenced-code-blocks.ts diff --git a/libs/mobile/shared/data-access/speech-streaming-service/src/remove-fenced-code-blocks.ts b/libs/mobile/shared/data-access/speech-streaming-service/src/remove-fenced-code-blocks.ts new file mode 100644 index 00000000..f7f87ac1 --- /dev/null +++ b/libs/mobile/shared/data-access/speech-streaming-service/src/remove-fenced-code-blocks.ts @@ -0,0 +1,145 @@ +export interface RemoveFencedCodeBlocksOptions { + holdIncomplete?: boolean; +} + +/** + * Removes fenced markdown code blocks (``` / ~~~) from text for TTS. + * When holdIncomplete is true, truncates from an unclosed opening fence + * so streaming code is not spoken before the fence closes. + */ +export const removeFencedCodeBlocks = ( + text: string, + { holdIncomplete = false }: RemoveFencedCodeBlocksOptions = {}, +): string => { + let result = ''; + let index = 0; + + while (index < text.length) { + const atLineStart = index === 0 || text[index - 1] === '\n'; + + if (atLineStart) { + const fenceMatch = matchOpeningFence(text, index); + + if (fenceMatch) { + const closingEnd = findClosingFence(text, fenceMatch); + + if (closingEnd !== null) { + result += result.length > 0 && !/\s$/.test(result) ? ' ' : ''; + index = closingEnd; + + continue; + } + + if (holdIncomplete) { + return result; + } + + return result.length > 0 && !/\s$/.test(result) ? `${result} ` : result; + } + } + + result += text[index]; + index += 1; + } + + return result; +}; + +interface OpeningFenceMatch { + start: number; + fenceChar: '`' | '~'; + fenceLength: number; + contentStart: number; +} + +const matchOpeningFence = (text: string, start: number): OpeningFenceMatch | null => { + let index = start; + let spaces = 0; + + while (spaces < 3 && text[index] === ' ') { + spaces += 1; + index += 1; + } + + const fenceChar = text[index]; + + if (fenceChar !== '`' && fenceChar !== '~') { + return null; + } + + let fenceLength = 0; + + while (text[index + fenceLength] === fenceChar) { + fenceLength += 1; + } + + if (fenceLength < 3) { + return null; + } + + const infoStart = index + fenceLength; + const lineEnd = text.indexOf('\n', infoStart); + + // Incomplete opening fence line — treat as an open fence + if (lineEnd === -1) { + return { + start, + fenceChar, + fenceLength, + contentStart: text.length, + }; + } + + return { + start, + fenceChar, + fenceLength, + contentStart: lineEnd + 1, + }; +}; + +const findClosingFence = (text: string, opening: OpeningFenceMatch): number | null => { + if (opening.contentStart >= text.length) { + return null; + } + + let lineStart = opening.contentStart; + + while (lineStart <= text.length) { + let index = lineStart; + let spaces = 0; + + while (spaces < 3 && text[index] === ' ') { + spaces += 1; + index += 1; + } + + if (text[index] === opening.fenceChar) { + let closeLength = 0; + + while (text[index + closeLength] === opening.fenceChar) { + closeLength += 1; + } + + if (closeLength >= opening.fenceLength) { + const afterFence = index + closeLength; + const nextNewline = text.indexOf('\n', afterFence); + const restOfLine = nextNewline === -1 ? text.slice(afterFence) : text.slice(afterFence, nextNewline); + + if (/^\s*$/.test(restOfLine)) { + return nextNewline === -1 ? text.length : nextNewline + 1; + } + } + } + + const nextNewline = text.indexOf('\n', lineStart); + + if (nextNewline === -1) { + return null; + } + + lineStart = nextNewline + 1; + } + + return null; +}; diff --git a/libs/mobile/shared/data-access/speech-streaming-service/src/service.ts b/libs/mobile/shared/data-access/speech-streaming-service/src/service.ts index c8a7b6c4..8e833954 100644 --- a/libs/mobile/shared/data-access/speech-streaming-service/src/service.ts +++ b/libs/mobile/shared/data-access/speech-streaming-service/src/service.ts @@ -1,6 +1,7 @@ import { setAudioModeAsync } from 'expo-audio'; import * as Speech from 'expo-speech'; import { SpeechStreamingServiceEvent } from './enums'; +import { removeFencedCodeBlocks } from './remove-fenced-code-blocks'; const textBreakpoints = ['.', '!', '?', ',', ';', ':', '-']; @@ -26,7 +27,9 @@ class SpeechStreamingService { }; public handleContent(text: string, isDone?: boolean): void { - const unspokenText = text.slice(this.spokenText.length); + // NOTE: Speak from text with code fences removed; cursor tracks speakable length + const speakableText = removeFencedCodeBlocks(text, { holdIncomplete: !isDone }); + const unspokenText = speakableText.slice(this.spokenText.length); let textToSpeak = ''; From 396adcb52674352c568da051d2c8db264f91c406 Mon Sep 17 00:00:00 2001 From: Mikhail Leonov Date: Tue, 4 Aug 2026 12:15:44 +0600 Subject: [PATCH 2/6] feat: improve speech streaming by integrating text preparation and refining handling of completed messages --- .../src/prepare-speakable-text.ts | 138 ++++++++++++++++++ .../speech-streaming-service/src/service.ts | 27 ++-- 2 files changed, 155 insertions(+), 10 deletions(-) create mode 100644 libs/mobile/shared/data-access/speech-streaming-service/src/prepare-speakable-text.ts diff --git a/libs/mobile/shared/data-access/speech-streaming-service/src/prepare-speakable-text.ts b/libs/mobile/shared/data-access/speech-streaming-service/src/prepare-speakable-text.ts new file mode 100644 index 00000000..0e4b9bac --- /dev/null +++ b/libs/mobile/shared/data-access/speech-streaming-service/src/prepare-speakable-text.ts @@ -0,0 +1,138 @@ +import { removeFencedCodeBlocks } from './remove-fenced-code-blocks'; + +export interface PrepareSpeakableTextOptions { + holdIncomplete?: boolean; +} + +/** + * Prepares streaming markdown for TTS: drops code fences, details/tool blocks, + * math, citations and markdown syntax while keeping readable prose. + */ +export const prepareSpeakableText = ( + text: string, + { holdIncomplete = false }: PrepareSpeakableTextOptions = {}, +): string => { + let result = removeFencedCodeBlocks(text, { holdIncomplete }); + result = removeDetailsBlocks(result, { holdIncomplete }); + result = stripMarkdownForSpeech(result); + result = normalizeSpeakableWhitespace(result); + + return result; +}; + +const removeDetailsBlocks = ( + text: string, + { holdIncomplete = false }: PrepareSpeakableTextOptions, +): string => { + let result = ''; + let index = 0; + + while (index < text.length) { + const openMatch = text.slice(index).match(/<\s*details\b/i); + + if (!openMatch || openMatch.index === undefined) { + result += text.slice(index); + break; + } + + const openStart = index + openMatch.index; + result += text.slice(index, openStart); + + const fromOpen = text.slice(openStart); + const openEndOffset = indexAfterOpenTag(fromOpen); + + if (openEndOffset === -1) { + return holdIncomplete ? result : appendBoundarySpace(result); + } + + const closeMatch = fromOpen.slice(openEndOffset).match(/<\s*\/\s*details\s*>/i); + + if (!closeMatch || closeMatch.index === undefined) { + return holdIncomplete ? result : appendBoundarySpace(result); + } + + index = openStart + openEndOffset + closeMatch.index + closeMatch[0].length; + result = appendBoundarySpace(result); + } + + return result; +}; + +const indexAfterOpenTag = (text: string): number => { + let index = 0; + let inDouble = false; + let escape = false; + + while (index < text.length) { + const char = text[index]; + + if (escape) { + escape = false; + index += 1; + continue; + } + + if (char === '\\') { + escape = true; + index += 1; + continue; + } + + if (char === '"') { + inDouble = !inDouble; + index += 1; + continue; + } + + if (char === '>' && !inDouble) { + return index + 1; + } + + index += 1; + } + + return -1; +}; + +const stripMarkdownForSpeech = (text: string): string => { + let result = text; + + // NOTE: Display math first so inline $ patterns do not match block delimiters + result = result.replace(/\$\$[\s\S]*?\$\$/g, ' '); + result = result.replace(/\$([^$\n]+)\$/g, ' '); + + // NOTE: Inline code (fenced blocks already removed) + result = result.replace(/`[^`\n]+`/g, ' '); + + // NOTE: Images and links — keep visible label only + result = result.replace(/!\[([^\]]*)\]\([^)]*\)/g, '$1'); + result = result.replace(/\[([^\]]+)\]\([^)]*\)/g, '$1'); + + // NOTE: Bold / italic / strike — skip single '_' to preserve snake_case + result = result.replace(/\*\*(.+?)\*\*/g, '$1'); + result = result.replace(/__(.+?)__/g, '$1'); + result = result.replace(/\*(.+?)\*/g, '$1'); + result = result.replace(/~~(.+?)~~/g, '$1'); + + // NOTE: ATX headings and list markers at line start + result = result.replace(/^#{1,6}\s+/gm, ''); + result = result.replace(/^\s*([-*+]|\d+\.)\s+/gm, ''); + + // NOTE: Citation markers like [1], [12] + result = result.replace(/\[(\d+)\]/g, ' '); + + // NOTE: Horizontal rules + result = result.replace(/^\s{0,3}([-*_])(?:\s*\1){2,}\s*$/gm, ' '); + + // NOTE: Blockquote markers + result = result.replace(/^\s{0,3}>\s?/gm, ''); + + return result; +}; + +// NOTE: Avoid trim() so streaming speakable prefixes stay stable for the spoken cursor +const normalizeSpeakableWhitespace = (text: string): string => + text.replace(/[^\S\n]+/g, ' ').replace(/\n{3,}/g, '\n\n').replace(/ ?\n ?/g, '\n'); + +const appendBoundarySpace = (text: string): string => + text.length > 0 && !/\s$/.test(text) ? `${text} ` : text; diff --git a/libs/mobile/shared/data-access/speech-streaming-service/src/service.ts b/libs/mobile/shared/data-access/speech-streaming-service/src/service.ts index 8e833954..41520c83 100644 --- a/libs/mobile/shared/data-access/speech-streaming-service/src/service.ts +++ b/libs/mobile/shared/data-access/speech-streaming-service/src/service.ts @@ -1,7 +1,7 @@ import { setAudioModeAsync } from 'expo-audio'; import * as Speech from 'expo-speech'; import { SpeechStreamingServiceEvent } from './enums'; -import { removeFencedCodeBlocks } from './remove-fenced-code-blocks'; +import { prepareSpeakableText } from './prepare-speakable-text'; const textBreakpoints = ['.', '!', '?', ',', ';', ':', '-']; @@ -27,17 +27,22 @@ class SpeechStreamingService { }; public handleContent(text: string, isDone?: boolean): void { - // NOTE: Speak from text with code fences removed; cursor tracks speakable length - const speakableText = removeFencedCodeBlocks(text, { holdIncomplete: !isDone }); + // NOTE: Speak cleaned markdown; cursor tracks speakable length + const speakableText = prepareSpeakableText(text, { holdIncomplete: !isDone }); const unspokenText = speakableText.slice(this.spokenText.length); let textToSpeak = ''; - // NOTE: We need to separate text by breakpoints to make it more natural - for (let i = unspokenText.length - 1; i >= 0; i--) { - if (textBreakpoints.includes(unspokenText[i])) { - textToSpeak = unspokenText.slice(0, i + 1); - break; + if (isDone) { + // NOTE: Flush all leftover speakable text when the message is complete + textToSpeak = unspokenText; + } else { + // NOTE: Separate text by breakpoints to make streaming speech more natural + for (let i = unspokenText.length - 1; i >= 0; i--) { + if (textBreakpoints.includes(unspokenText[i])) { + textToSpeak = unspokenText.slice(0, i + 1); + break; + } } } @@ -48,11 +53,13 @@ class SpeechStreamingService { this.spokenText = this.spokenText + textToSpeak; - this.speakText(textToSpeak); + this.speakText(textToSpeak, isDone); + + return; } if (isDone) { - // NOTE: If the text is done, we need to stop speaking and emit the event + // NOTE: All speakable text was already queued; emit end after the speech queue drains this.speakText('', true); } } From 9b8e00facb1a8a6782d10aea29e68e08549e3dd1 Mon Sep 17 00:00:00 2001 From: Mikhail Leonov Date: Tue, 4 Aug 2026 12:26:58 +0600 Subject: [PATCH 3/6] feat: enhance chat creation and voice mode handling with improved state management and caching --- apps/mobile/app/(main)/chat/_layout.tsx | 3 +- .../src/use-create-new-chat.ts | 12 ++++++ .../voice-mode-modal/src/lib/component.tsx | 40 ++++++++++++++----- 3 files changed, 43 insertions(+), 12 deletions(-) diff --git a/apps/mobile/app/(main)/chat/_layout.tsx b/apps/mobile/app/(main)/chat/_layout.tsx index 1e60e993..b9cbb50d 100644 --- a/apps/mobile/app/(main)/chat/_layout.tsx +++ b/apps/mobile/app/(main)/chat/_layout.tsx @@ -22,7 +22,8 @@ export default function ChatLayout(): ReactElement { close: async () => await voiceModeModalRef.current?.close(), }; - const handleChatCreated = (id: string): void => router.push(navigationConfig.main.chat.view({ id })); + const handleChatCreated = (id: string): void => + router.push(navigationConfig.main.chat.view({ id, isNewChat: 'true' })); return ( diff --git a/libs/mobile/chat/features/use-create-new-chat/src/use-create-new-chat.ts b/libs/mobile/chat/features/use-create-new-chat/src/use-create-new-chat.ts index c56dbe8b..3e6ed5a4 100644 --- a/libs/mobile/chat/features/use-create-new-chat/src/use-create-new-chat.ts +++ b/libs/mobile/chat/features/use-create-new-chat/src/use-create-new-chat.ts @@ -2,11 +2,14 @@ import dayjs from 'dayjs'; import { chatApi, ChatGenerationOption, + chatQueriesKeys, + ChatResponse, patchChatList, prepareCompleteChatPayload, prepareCreateChatPayload, } from '@open-webui-react-native/shared/data-access/api'; import { FileData, ImageData } from '@open-webui-react-native/shared/data-access/common'; +import { queryClient } from '@open-webui-react-native/shared/data-access/query-client'; import { socketService } from '@open-webui-react-native/shared/data-access/websocket'; interface UseCreateNewChatArgs { @@ -32,6 +35,15 @@ export function useCreateNewChat({ onSuccess }: UseCreateNewChatArgs): typeof re createNewChat(payload, { onSuccess: (data) => { + // NOTE: Seed get-chat cache so socket streaming and VoiceMode can track the assistant reply + const assistantMessage = data.chat.history.messages[data.chat.history.currentId]; + + if (assistantMessage) { + assistantMessage.done = false; + } + + queryClient.setQueryData(chatQueriesKeys.get(data.id).queryKey, data); + onSuccess?.(data.id); patchChatList({ id: data.id, diff --git a/libs/mobile/chat/features/voice-mode-modal/src/lib/component.tsx b/libs/mobile/chat/features/voice-mode-modal/src/lib/component.tsx index bb7e8923..70be9b91 100644 --- a/libs/mobile/chat/features/voice-mode-modal/src/lib/component.tsx +++ b/libs/mobile/chat/features/voice-mode-modal/src/lib/component.tsx @@ -43,6 +43,11 @@ export function VoiceModeModal({ onChatCreated, ref, ...props }: VoiceModeModalP const [chatId, setChatId] = useState(undefined); const [modelId, setModelId] = useState(''); + const chatIdRef = useRef(chatId); + const modelIdRef = useRef(modelId); + chatIdRef.current = chatId; + modelIdRef.current = modelId; + const handleChatCreated = (id: string): void => { if (isVisible) { setChatId(id); @@ -54,15 +59,20 @@ export function VoiceModeModal({ onChatCreated, ref, ...props }: VoiceModeModalP const { sendMessage, isLoading: isSending } = useSendMessage({ chatData: chat }); const { startChatCreation, isLoading: isCreating } = useCreateNewChat({ onSuccess: handleChatCreated }); + const sendMessageRef = useRef(sendMessage); + const startChatCreationRef = useRef(startChatCreation); + sendMessageRef.current = sendMessage; + startChatCreationRef.current = startChatCreation; + const { isTranscribing, startSpeechRecording, stopSpeechRecording, completeSpeechRecording, metering } = useDictateMode({ updateIntervalMillis: 100, onCompleteRecording: (text: string) => { if (text.trim().length) { - if (chatId) { - sendMessage(text, modelId); + if (chatIdRef.current) { + sendMessageRef.current(text, modelIdRef.current); } else { - startChatCreation(text, modelId); + startChatCreationRef.current(text, modelIdRef.current); } setIsWaitingNewMessage(true); @@ -137,20 +147,28 @@ export function VoiceModeModal({ onChatCreated, ref, ...props }: VoiceModeModalP }, [isVisible]); useEffect(() => { - if (isVisible) { - if (isWaitingNewMessage && newMessage && !newMessage.done) { - // NOTE: In this case, we start receiving a new message via WebSocket + if (!isVisible) { + return; + } + + if (isWaitingNewMessage && newMessage) { + if (!newMessage.done) { + // NOTE: Start receiving a new message via WebSocket setIsWaitingNewMessage(false); setIsReceivingNewMessage(true); speechStreamingService.handleContent(newMessage.content); + } else if (newMessage.content.trim()) { + // NOTE: Reply already finished before streaming subscription (common on create-chat) + setIsWaitingNewMessage(false); + speechStreamingService.handleContent(newMessage.content, true); } + } - if (isReceivingNewMessage && newMessage) { - speechStreamingService.handleContent(newMessage.content, newMessage.done); + if (isReceivingNewMessage && newMessage) { + speechStreamingService.handleContent(newMessage.content, newMessage.done); - if (newMessage.done) { - setIsReceivingNewMessage(false); - } + if (newMessage.done) { + setIsReceivingNewMessage(false); } } }, [isVisible, isWaitingNewMessage, isReceivingNewMessage, newMessage?.content.length, newMessage?.done]); From baabf604440950ee58e99b66a2913473a6539507 Mon Sep 17 00:00:00 2001 From: Mikhail Leonov Date: Tue, 4 Aug 2026 15:51:21 +0600 Subject: [PATCH 4/6] feat: improve voice mode functionality by adding content resumption and enhanced stop handling --- .../voice-mode-modal/src/lib/component.tsx | 8 ++++-- .../speech-streaming-service/src/service.ts | 28 ++++++++++++++++++- 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/libs/mobile/chat/features/voice-mode-modal/src/lib/component.tsx b/libs/mobile/chat/features/voice-mode-modal/src/lib/component.tsx index 70be9b91..b64827a4 100644 --- a/libs/mobile/chat/features/voice-mode-modal/src/lib/component.tsx +++ b/libs/mobile/chat/features/voice-mode-modal/src/lib/component.tsx @@ -75,6 +75,7 @@ export function VoiceModeModal({ onChatCreated, ref, ...props }: VoiceModeModalP startChatCreationRef.current(text, modelIdRef.current); } + speechStreamingService.resumeContentSpeaking(); setIsWaitingNewMessage(true); } else { startSpeechRecording(); @@ -87,14 +88,17 @@ export function VoiceModeModal({ onChatCreated, ref, ...props }: VoiceModeModalP isCreating || isSending || isLoading || isTranscribing || isWaitingNewMessage || isReceivingNewMessage; const close = async (): Promise => { - await stopSpeechRecording(); - await speechStreamingService.stopContentSpeaking(); + // NOTE: Stop TTS immediately; isStopped is set sync so late handleContent/speakText no-ops + const stopSpeakingPromise = speechStreamingService.stopContentSpeaking(); + speechStreamingService.clearListeners(); clearSilenceTimeout(); setIsUserSpeaking(false); setIsAiSpeaking(false); setIsWaitingNewMessage(false); setIsReceivingNewMessage(false); setIsVisible(false); + await stopSpeakingPromise; + await stopSpeechRecording(); }; useImperativeHandle( diff --git a/libs/mobile/shared/data-access/speech-streaming-service/src/service.ts b/libs/mobile/shared/data-access/speech-streaming-service/src/service.ts index 41520c83..60d27940 100644 --- a/libs/mobile/shared/data-access/speech-streaming-service/src/service.ts +++ b/libs/mobile/shared/data-access/speech-streaming-service/src/service.ts @@ -7,10 +7,12 @@ const textBreakpoints = ['.', '!', '?', ',', ';', ':', '-']; class SpeechStreamingService { private spokenText: string; + private isStopped: boolean; private listeners: Map) => void>> = new Map(); constructor() { this.spokenText = ''; + this.isStopped = true; } public onSpeakingStart(callback: () => void): () => void { @@ -21,12 +23,23 @@ class SpeechStreamingService { return this.addEventListener(SpeechStreamingServiceEvent.SPEAKING_END, callback); } + public resumeContentSpeaking = (): void => { + this.isStopped = false; + this.spokenText = ''; + }; + public stopContentSpeaking = async (): Promise => { - await Speech.stop(); + // NOTE: Set before await so in-flight speakText calls bail out after setAudioModeAsync + this.isStopped = true; this.spokenText = ''; + await Speech.stop(); }; public handleContent(text: string, isDone?: boolean): void { + if (this.isStopped) { + return; + } + // NOTE: Speak cleaned markdown; cursor tracks speakable length const speakableText = prepareSpeakableText(text, { holdIncomplete: !isDone }); const unspokenText = speakableText.slice(this.spokenText.length); @@ -69,15 +82,28 @@ class SpeechStreamingService { } private speakText = async (text: string, isDone?: boolean): Promise => { + if (this.isStopped) { + return; + } + // NOTE: Need to set audio mode to allow speech in silent mode on iOS await setAudioModeAsync({ playsInSilentMode: true, }); + // NOTE: stopContentSpeaking may have been called while awaiting audio mode + if (this.isStopped) { + return; + } + Speech.speak(text, { // NOTE: Only English is working good for now language: 'en-US', onDone: () => { + if (this.isStopped) { + return; + } + if (isDone) { this.emit(SpeechStreamingServiceEvent.SPEAKING_END); this.spokenText = ''; From bc7efdbdad3efd23f450a5762047fbfbdb8832f7 Mon Sep 17 00:00:00 2001 From: Mikhail Leonov Date: Thu, 6 Aug 2026 12:12:40 +0600 Subject: [PATCH 5/6] feat: integrate expo-camera for enhanced voice mode functionality with image capture support --- apps/mobile/app.config.ts | 8 ++ apps/mobile/package.json | 1 + .../voice-mode-modal/src/lib/component.tsx | 84 ++++++++++++++++--- .../components/camera-preview/component.tsx | 73 ++++++++++++++++ .../lib/components/camera-preview/index.ts | 1 + .../src/lib/components/index.ts | 1 + .../voice-mode-modal/src/lib/config.ts | 4 +- package-lock.json | 77 +++++++++++++++++ package.json | 1 + 9 files changed, 237 insertions(+), 13 deletions(-) create mode 100644 libs/mobile/chat/features/voice-mode-modal/src/lib/components/camera-preview/component.tsx create mode 100644 libs/mobile/chat/features/voice-mode-modal/src/lib/components/camera-preview/index.ts diff --git a/apps/mobile/app.config.ts b/apps/mobile/app.config.ts index 09be2b02..c6127898 100644 --- a/apps/mobile/app.config.ts +++ b/apps/mobile/app.config.ts @@ -99,6 +99,14 @@ const createConfig = (): Omit & { extra: { eas: EASConfig } 'Open MobileUI uses your camera to let you take photos and share them directly in chat conversations.', }, ], + [ + 'expo-camera', + { + cameraPermission: + 'Open MobileUI uses your camera to let you share live visuals during voice mode conversations.', + recordAudioAndroid: false, + }, + ], [ 'expo-media-library', { diff --git a/apps/mobile/package.json b/apps/mobile/package.json index ea885eda..e46da6cd 100644 --- a/apps/mobile/package.json +++ b/apps/mobile/package.json @@ -52,6 +52,7 @@ "expo-asset": "~57.0.7", "expo-audio": "~57.0.3", "expo-build-properties": "~57.0.7", + "expo-camera": "~57.0.3", "expo-clipboard": "~57.0.1", "expo-constants": "~57.0.7", "expo-crypto": "~57.0.1", diff --git a/libs/mobile/chat/features/voice-mode-modal/src/lib/component.tsx b/libs/mobile/chat/features/voice-mode-modal/src/lib/component.tsx index b64827a4..ac4672d0 100644 --- a/libs/mobile/chat/features/voice-mode-modal/src/lib/component.tsx +++ b/libs/mobile/chat/features/voice-mode-modal/src/lib/component.tsx @@ -1,4 +1,5 @@ -import { useTranslation } from '@ronas-it/react-native-common-modules/i18n'; +import { i18n, useTranslation } from '@ronas-it/react-native-common-modules/i18n'; +import { CameraType, useCameraPermissions } from 'expo-camera'; import { ForwardedRef, ReactElement, useEffect, useImperativeHandle, useRef, useState } from 'react'; import Modal, { ModalProps } from 'react-native-modal'; import { useCreateNewChat } from '@open-webui-react-native/mobile/chat/features/use-create-new-chat'; @@ -8,8 +9,9 @@ import { useDictateMode } from '@open-webui-react-native/mobile/shared/features/ import { colors, useColorScheme } from '@open-webui-react-native/mobile/shared/ui/styles'; import { AppSafeAreaView, AppText, AppToast, IconButton, View } from '@open-webui-react-native/mobile/shared/ui/ui-kit'; import { chatApi } from '@open-webui-react-native/shared/data-access/api'; +import { ImageData as ChatImageData } from '@open-webui-react-native/shared/data-access/common'; import { ToastService } from '@open-webui-react-native/shared/utils/toast-service'; -import { Loader, SpeechListener } from './components'; +import { CameraPreview, CameraPreviewMethods, Loader, SpeechListener } from './components'; import { voiceModeModalConfig } from './config'; export type VoiceModeModalMethods = { @@ -29,8 +31,12 @@ const { meteringSilenceThreshold, meteringSilenceDuration } = voiceModeModalConf export function VoiceModeModal({ onChatCreated, ref, ...props }: VoiceModeModalProps): ReactElement { const translate = useTranslation('CHAT.VOICE_MODE_MODAL'); const { isDarkColorScheme } = useColorScheme(); + const [, requestCameraPermission] = useCameraPermissions(); const silenceTimeout = useRef | null>(null); + const cameraPreviewRef = useRef(null); + const pendingImageRef = useRef(null); + const isCameraOnRef = useRef(false); const [isVisible, setIsVisible] = useState(false); @@ -43,10 +49,14 @@ export function VoiceModeModal({ onChatCreated, ref, ...props }: VoiceModeModalP const [chatId, setChatId] = useState(undefined); const [modelId, setModelId] = useState(''); + const [isCameraOn, setIsCameraOn] = useState(false); + const [cameraFacing, setCameraFacing] = useState('front'); + const chatIdRef = useRef(chatId); const modelIdRef = useRef(modelId); chatIdRef.current = chatId; modelIdRef.current = modelId; + isCameraOnRef.current = isCameraOn; const handleChatCreated = (id: string): void => { if (isVisible) { @@ -68,11 +78,14 @@ export function VoiceModeModal({ onChatCreated, ref, ...props }: VoiceModeModalP useDictateMode({ updateIntervalMillis: 100, onCompleteRecording: (text: string) => { + const attachedImages = pendingImageRef.current ? [pendingImageRef.current] : undefined; + pendingImageRef.current = null; + if (text.trim().length) { if (chatIdRef.current) { - sendMessageRef.current(text, modelIdRef.current); + sendMessageRef.current(text, modelIdRef.current, undefined, undefined, attachedImages); } else { - startChatCreationRef.current(text, modelIdRef.current); + startChatCreationRef.current(text, modelIdRef.current, undefined, undefined, attachedImages); } speechStreamingService.resumeContentSpeaking(); @@ -87,11 +100,17 @@ export function VoiceModeModal({ onChatCreated, ref, ...props }: VoiceModeModalP const isThinking = isCreating || isSending || isLoading || isTranscribing || isWaitingNewMessage || isReceivingNewMessage; + const stopCamera = (): void => { + setIsCameraOn(false); + }; + const close = async (): Promise => { // NOTE: Stop TTS immediately; isStopped is set sync so late handleContent/speakText no-ops const stopSpeakingPromise = speechStreamingService.stopContentSpeaking(); speechStreamingService.clearListeners(); clearSilenceTimeout(); + stopCamera(); + pendingImageRef.current = null; setIsUserSpeaking(false); setIsAiSpeaking(false); setIsWaitingNewMessage(false); @@ -116,7 +135,31 @@ export function VoiceModeModal({ onChatCreated, ref, ...props }: VoiceModeModalP [], ); - const showUnderConstruction = (): void => ToastService.showFeatureNotImplemented(); + const startCamera = async (): Promise => { + const permission = await requestCameraPermission(); + + if (!permission.granted) { + ToastService.showError(i18n.t('SHARED.IMAGE_PICKER_SERVICE.TEXT_ACCESS_DENIED')); + + return; + } + + setIsCameraOn(true); + }; + + const flipCameraFacing = (): void => { + setCameraFacing((current) => (current === 'back' ? 'front' : 'back')); + }; + + const capturePendingImage = async (): Promise => { + if (!isCameraOnRef.current) { + pendingImageRef.current = null; + + return; + } + + pendingImageRef.current = (await cameraPreviewRef.current?.takePicture()) ?? null; + }; const clearSilenceTimeout = (): void => { if (silenceTimeout.current) { @@ -131,8 +174,11 @@ export function VoiceModeModal({ onChatCreated, ref, ...props }: VoiceModeModalP } silenceTimeout.current = setTimeout(() => { - setIsUserSpeaking(false); - completeSpeechRecording(); + void (async () => { + setIsUserSpeaking(false); + await capturePendingImage(); + await completeSpeechRecording(); + })(); }, meteringSilenceDuration); }; @@ -210,13 +256,29 @@ export function VoiceModeModal({ onChatCreated, ref, ...props }: VoiceModeModalP {...props}> - - {isThinking || isAiSpeaking ? : } + + {isCameraOn ? ( + + + {(isThinking || isAiSpeaking) && ( + + + + )} + + ) : isThinking || isAiSpeaking ? ( + + ) : ( + + )} diff --git a/libs/mobile/chat/features/voice-mode-modal/src/lib/components/camera-preview/component.tsx b/libs/mobile/chat/features/voice-mode-modal/src/lib/components/camera-preview/component.tsx new file mode 100644 index 00000000..54b97267 --- /dev/null +++ b/libs/mobile/chat/features/voice-mode-modal/src/lib/components/camera-preview/component.tsx @@ -0,0 +1,73 @@ +import { CameraType, CameraView } from 'expo-camera'; +import { ForwardedRef, ReactElement, useImperativeHandle, useRef, useState } from 'react'; +import { StyleSheet } from 'react-native'; +import { IconButton, View } from '@open-webui-react-native/mobile/shared/ui/ui-kit'; +import { ImageData as ChatImageData } from '@open-webui-react-native/shared/data-access/common'; + +export type CameraPreviewMethods = { + takePicture: () => Promise; +}; + +export type CameraPreviewRef = ForwardedRef; + +export interface CameraPreviewProps { + facing: CameraType; + onClose: () => void; + ref?: CameraPreviewRef; +} + +const PICTURE_QUALITY = 0.2; + +export function CameraPreview({ facing, onClose, ref }: CameraPreviewProps): ReactElement { + const cameraRef = useRef(null); + const [isReady, setIsReady] = useState(false); + + useImperativeHandle( + ref, + () => ({ + takePicture: async (): Promise => { + if (!isReady || !cameraRef.current) { + return null; + } + + try { + const photo = await cameraRef.current.takePictureAsync({ + base64: true, + quality: PICTURE_QUALITY, + shutterSound: false, + }); + + if (!photo?.uri || !photo.base64) { + return null; + } + + return { + uri: photo.uri, + base64: photo.base64, + mimeType: 'image/jpeg', + }; + } catch { + return null; + } + }, + }), + [isReady], + ); + + return ( + + setIsReady(true)} + /> + + + ); +} diff --git a/libs/mobile/chat/features/voice-mode-modal/src/lib/components/camera-preview/index.ts b/libs/mobile/chat/features/voice-mode-modal/src/lib/components/camera-preview/index.ts new file mode 100644 index 00000000..bb824842 --- /dev/null +++ b/libs/mobile/chat/features/voice-mode-modal/src/lib/components/camera-preview/index.ts @@ -0,0 +1 @@ +export * from './component'; diff --git a/libs/mobile/chat/features/voice-mode-modal/src/lib/components/index.ts b/libs/mobile/chat/features/voice-mode-modal/src/lib/components/index.ts index 0c4065d0..1673dfaf 100644 --- a/libs/mobile/chat/features/voice-mode-modal/src/lib/components/index.ts +++ b/libs/mobile/chat/features/voice-mode-modal/src/lib/components/index.ts @@ -1,2 +1,3 @@ +export * from './camera-preview'; export * from './loader'; export * from './speech-listener'; diff --git a/libs/mobile/chat/features/voice-mode-modal/src/lib/config.ts b/libs/mobile/chat/features/voice-mode-modal/src/lib/config.ts index 1bd4afbe..459e7f20 100644 --- a/libs/mobile/chat/features/voice-mode-modal/src/lib/config.ts +++ b/libs/mobile/chat/features/voice-mode-modal/src/lib/config.ts @@ -1,4 +1,4 @@ export const voiceModeModalConfig = { - meteringSilenceThreshold: 0.3, - meteringSilenceDuration: 2500, + meteringSilenceThreshold: 0.5, + meteringSilenceDuration: 1500, }; diff --git a/package-lock.json b/package-lock.json index e7045e59..5786e6b2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -41,6 +41,7 @@ "expo-asset": "~57.0.7", "expo-audio": "~57.0.3", "expo-build-properties": "~57.0.7", + "expo-camera": "~57.0.3", "expo-clipboard": "~57.0.1", "expo-constants": "~57.0.7", "expo-crypto": "~57.0.1", @@ -196,6 +197,7 @@ "expo-asset": "~57.0.7", "expo-audio": "~57.0.3", "expo-build-properties": "~57.0.7", + "expo-camera": "~57.0.3", "expo-clipboard": "~57.0.1", "expo-constants": "~57.0.7", "expo-crypto": "~57.0.1", @@ -10704,6 +10706,12 @@ "@types/node": "*" } }, + "node_modules/@types/emscripten": { + "version": "1.41.5", + "resolved": "https://registry.npmjs.org/@types/emscripten/-/emscripten-1.41.5.tgz", + "integrity": "sha512-cMQm7pxu6BxtHyqJ7mQZ2kXWV5SLmugybFdHCBbJ5eHzOo6VhBckEgAT3//rP5FwPHNPeEiq4SmQ5ucBwsOo4Q==", + "license": "MIT" + }, "node_modules/@types/estree": { "version": "1.0.9", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.9.tgz", @@ -13160,6 +13168,15 @@ "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", "license": "MIT" }, + "node_modules/barcode-detector": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/barcode-detector/-/barcode-detector-3.2.1.tgz", + "integrity": "sha512-zLL7AbT9uNJBUzYpKg9v5tUA4yQGReExSi60q0g660Mj0wjUhKmN0GrUF3qELo8ITW9THzyJXdZQkXLMnsieiw==", + "license": "MIT", + "dependencies": { + "zxing-wasm": "3.1.1" + } + }, "node_modules/base-64": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/base-64/-/base-64-1.0.0.tgz", @@ -16982,6 +16999,26 @@ "node": ">=10" } }, + "node_modules/expo-camera": { + "version": "57.0.3", + "resolved": "https://registry.npmjs.org/expo-camera/-/expo-camera-57.0.3.tgz", + "integrity": "sha512-Q+3aZ63eQCkdB6/FZrO/lfacNAg/j8JCeKQL2nBdf6vBeOo1Y2PKYx1/vK+U5LaRnIo/0tMGmCOzZ1JGhTeMIw==", + "license": "MIT", + "dependencies": { + "barcode-detector": "^3.0.0" + }, + "peerDependencies": { + "expo": "*", + "react": "*", + "react-native": "*", + "react-native-web": "*" + }, + "peerDependenciesMeta": { + "react-native-web": { + "optional": true + } + } + }, "node_modules/expo-clipboard": { "version": "57.0.1", "resolved": "https://registry.npmjs.org/expo-clipboard/-/expo-clipboard-57.0.1.tgz", @@ -31503,6 +31540,18 @@ "url": "https://github.com/chalk/strip-ansi?sponsor=1" } }, + "node_modules/tagged-tag": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/tagged-tag/-/tagged-tag-1.0.0.tgz", + "integrity": "sha512-yEFYrVhod+hdNyx7g5Bnkkb0G6si8HJurOoOEgC8B/O0uXLHlaey/65KRv6cuWBNhBgHKAROVpc7QyYqE5gFng==", + "license": "MIT", + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/tailwind-merge": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/tailwind-merge/-/tailwind-merge-3.6.0.tgz", @@ -34082,6 +34131,34 @@ "funding": { "url": "https://github.com/sponsors/colinhacks" } + }, + "node_modules/zxing-wasm": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/zxing-wasm/-/zxing-wasm-3.1.1.tgz", + "integrity": "sha512-g0sPJBIubO6zLcJh1jftLPIN6xziaqLsvLgtpGKwDrEhyXXqla3E3yjFrznlr78UHIOMzbJPi0HDWKs/KgaB7A==", + "license": "MIT", + "dependencies": { + "@types/emscripten": "^1.41.5", + "type-fest": "^5.8.0" + }, + "peerDependencies": { + "@types/emscripten": ">=1.39.6" + } + }, + "node_modules/zxing-wasm/node_modules/type-fest": { + "version": "5.8.0", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-5.8.0.tgz", + "integrity": "sha512-YGYEVz3Fm5iy/AybuA0oyNFq7H4CgQNfRp/qfe8nurE1kuCeNm3/vfm9X4Mtl+qLyaKJUh5xrFZwogr41SMjYA==", + "license": "(MIT OR CC0-1.0)", + "dependencies": { + "tagged-tag": "^1.0.0" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } } } } diff --git a/package.json b/package.json index 7b532517..ac2ae070 100644 --- a/package.json +++ b/package.json @@ -39,6 +39,7 @@ "expo-asset": "~57.0.7", "expo-audio": "~57.0.3", "expo-build-properties": "~57.0.7", + "expo-camera": "~57.0.3", "expo-clipboard": "~57.0.1", "expo-constants": "~57.0.7", "expo-crypto": "~57.0.1", From 680c674a203160a7ebd25786ffde15bca3aa1932 Mon Sep 17 00:00:00 2001 From: Mikhail Leonov Date: Tue, 11 Aug 2026 10:39:10 +0600 Subject: [PATCH 6/6] feat: implement speech queue management and improve processing logic for speech segments --- .../speech-streaming-service/src/service.ts | 101 ++++++++++++++---- 1 file changed, 82 insertions(+), 19 deletions(-) diff --git a/libs/mobile/shared/data-access/speech-streaming-service/src/service.ts b/libs/mobile/shared/data-access/speech-streaming-service/src/service.ts index 60d27940..5ba2f1e7 100644 --- a/libs/mobile/shared/data-access/speech-streaming-service/src/service.ts +++ b/libs/mobile/shared/data-access/speech-streaming-service/src/service.ts @@ -5,14 +5,25 @@ import { prepareSpeakableText } from './prepare-speakable-text'; const textBreakpoints = ['.', '!', '?', ',', ';', ':', '-']; +type SpeechQueueItem = { + text: string; + isFinal: boolean; +}; + class SpeechStreamingService { private spokenText: string; private isStopped: boolean; + private speechGeneration: number; + private queue: Array; + private isProcessingQueue: boolean; private listeners: Map) => void>> = new Map(); constructor() { this.spokenText = ''; this.isStopped = true; + this.speechGeneration = 0; + this.queue = []; + this.isProcessingQueue = false; } public onSpeakingStart(callback: () => void): () => void { @@ -26,13 +37,17 @@ class SpeechStreamingService { public resumeContentSpeaking = (): void => { this.isStopped = false; this.spokenText = ''; + this.queue = []; }; public stopContentSpeaking = async (): Promise => { - // NOTE: Set before await so in-flight speakText calls bail out after setAudioModeAsync + // NOTE: Bump generation before await so in-flight onDone / queue work becomes stale + this.speechGeneration += 1; this.isStopped = true; this.spokenText = ''; + this.queue = []; await Speech.stop(); + this.isProcessingQueue = false; }; public handleContent(text: string, isDone?: boolean): void { @@ -65,15 +80,14 @@ class SpeechStreamingService { } this.spokenText = this.spokenText + textToSpeak; - - this.speakText(textToSpeak, isDone); + this.enqueue({ text: textToSpeak, isFinal: !!isDone }); return; } if (isDone) { // NOTE: All speakable text was already queued; emit end after the speech queue drains - this.speakText('', true); + this.enqueue({ text: '', isFinal: true }); } } @@ -81,34 +95,83 @@ class SpeechStreamingService { this.listeners.clear(); } - private speakText = async (text: string, isDone?: boolean): Promise => { + private enqueue(item: SpeechQueueItem): void { if (this.isStopped) { return; } - // NOTE: Need to set audio mode to allow speech in silent mode on iOS - await setAudioModeAsync({ - playsInSilentMode: true, - }); + this.queue.push(item); + void this.processQueue(); + } - // NOTE: stopContentSpeaking may have been called while awaiting audio mode - if (this.isStopped) { + private processQueue = async (): Promise => { + if (this.isProcessingQueue) { return; } - Speech.speak(text, { - // NOTE: Only English is working good for now - language: 'en-US', - onDone: () => { - if (this.isStopped) { - return; + this.isProcessingQueue = true; + const generation = this.speechGeneration; + + try { + while (this.queue.length > 0) { + if (this.isStopped || generation !== this.speechGeneration) { + break; + } + + const item = this.queue.shift(); + + if (!item) { + break; + } + + if (item.text.trim()) { + await this.speakSegment(item.text, generation); + } + + if (this.isStopped || generation !== this.speechGeneration) { + break; } - if (isDone) { + if (item.isFinal) { this.emit(SpeechStreamingServiceEvent.SPEAKING_END); this.spokenText = ''; } - }, + } + } finally { + if (generation === this.speechGeneration) { + this.isProcessingQueue = false; + } + } + }; + + private speakSegment = async (text: string, generation: number): Promise => { + if (this.isStopped || generation !== this.speechGeneration) { + return; + } + + // NOTE: Need to set audio mode to allow speech in silent mode on iOS + await setAudioModeAsync({ + playsInSilentMode: true, + }); + + if (this.isStopped || generation !== this.speechGeneration) { + return; + } + + await new Promise((resolve) => { + if (this.isStopped || generation !== this.speechGeneration) { + resolve(); + + return; + } + + Speech.speak(text, { + // NOTE: Only English is working good for now + language: 'en-US', + onDone: () => resolve(), + onStopped: () => resolve(), + onError: () => resolve(), + }); }); };