PRD-1946 - Adjust voice mode

apps/mobile/app.config.ts

@@ -99,6 +99,14 @@ const createConfig = (): Omit<ExpoConfig, 'extra'> & { extra: { eas: EASConfig }
             'Open MobileUI uses your camera to let you take photos and share them directly in chat conversations.',
         },
       ],
+      [
+        'expo-camera',
+        {
+          cameraPermission:
+            'Open MobileUI uses your camera to let you share live visuals during voice mode conversations.',
+          recordAudioAndroid: false,
+        },
+      ],
       [
         'expo-media-library',
         {

apps/mobile/app/(main)/chat/_layout.tsx

@@ -22,7 +22,8 @@ export default function ChatLayout(): ReactElement {
     close: async () => await voiceModeModalRef.current?.close(),
   };
 
-  const handleChatCreated = (id: string): void => router.push(navigationConfig.main.chat.view({ id }));
+  const handleChatCreated = (id: string): void =>
+    router.push(navigationConfig.main.chat.view({ id, isNewChat: 'true' }));
 
   return (
     <VoiceModeModalContext.Provider value={contextValue}>

apps/mobile/package.json

@@ -52,6 +52,7 @@
     "expo-asset": "~57.0.7",
     "expo-audio": "~57.0.3",
     "expo-build-properties": "~57.0.7",
+    "expo-camera": "~57.0.3",
     "expo-clipboard": "~57.0.1",
     "expo-constants": "~57.0.7",
     "expo-crypto": "~57.0.1",

libs/mobile/chat/features/use-create-new-chat/src/use-create-new-chat.ts

@@ -2,11 +2,14 @@ import dayjs from 'dayjs';
 import {
   chatApi,
   ChatGenerationOption,
+  chatQueriesKeys,
+  ChatResponse,
   patchChatList,
   prepareCompleteChatPayload,
   prepareCreateChatPayload,
 } from '@open-webui-react-native/shared/data-access/api';
 import { FileData, ImageData } from '@open-webui-react-native/shared/data-access/common';
+import { queryClient } from '@open-webui-react-native/shared/data-access/query-client';
 import { socketService } from '@open-webui-react-native/shared/data-access/websocket';
 
 interface UseCreateNewChatArgs {
@@ -32,6 +35,15 @@ export function useCreateNewChat({ onSuccess }: UseCreateNewChatArgs): typeof re
 
     createNewChat(payload, {
       onSuccess: (data) => {
+        // NOTE: Seed get-chat cache so socket streaming and VoiceMode can track the assistant reply
+        const assistantMessage = data.chat.history.messages[data.chat.history.currentId];
+
+        if (assistantMessage) {
+          assistantMessage.done = false;
+        }
+
+        queryClient.setQueryData<ChatResponse>(chatQueriesKeys.get(data.id).queryKey, data);
+
         onSuccess?.(data.id);
         patchChatList({
           id: data.id,

libs/mobile/chat/features/voice-mode-modal/src/lib/component.tsx

@@ -1,4 +1,5 @@
-import { useTranslation } from '@ronas-it/react-native-common-modules/i18n';
+import { i18n, useTranslation } from '@ronas-it/react-native-common-modules/i18n';
+import { CameraType, useCameraPermissions } from 'expo-camera';
 import { ForwardedRef, ReactElement, useEffect, useImperativeHandle, useRef, useState } from 'react';
 import Modal, { ModalProps } from 'react-native-modal';
 import { useCreateNewChat } from '@open-webui-react-native/mobile/chat/features/use-create-new-chat';
@@ -8,8 +9,9 @@ import { useDictateMode } from '@open-webui-react-native/mobile/shared/features/
 import { colors, useColorScheme } from '@open-webui-react-native/mobile/shared/ui/styles';
 import { AppSafeAreaView, AppText, AppToast, IconButton, View } from '@open-webui-react-native/mobile/shared/ui/ui-kit';
 import { chatApi } from '@open-webui-react-native/shared/data-access/api';
+import { ImageData as ChatImageData } from '@open-webui-react-native/shared/data-access/common';
 import { ToastService } from '@open-webui-react-native/shared/utils/toast-service';
-import { Loader, SpeechListener } from './components';
+import { CameraPreview, CameraPreviewMethods, Loader, SpeechListener } from './components';
 import { voiceModeModalConfig } from './config';
 
 export type VoiceModeModalMethods = {
@@ -29,8 +31,12 @@ const { meteringSilenceThreshold, meteringSilenceDuration } = voiceModeModalConf
 export function VoiceModeModal({ onChatCreated, ref, ...props }: VoiceModeModalProps): ReactElement {
   const translate = useTranslation('CHAT.VOICE_MODE_MODAL');
   const { isDarkColorScheme } = useColorScheme();
+  const [, requestCameraPermission] = useCameraPermissions();
 
   const silenceTimeout = useRef<ReturnType<typeof setTimeout> | null>(null);
+  const cameraPreviewRef = useRef<CameraPreviewMethods>(null);
+  const pendingImageRef = useRef<ChatImageData | null>(null);
+  const isCameraOnRef = useRef(false);
 
   const [isVisible, setIsVisible] = useState(false);
 
@@ -43,6 +49,15 @@ export function VoiceModeModal({ onChatCreated, ref, ...props }: VoiceModeModalP
   const [chatId, setChatId] = useState<string | undefined>(undefined);
   const [modelId, setModelId] = useState<string>('');
 
+  const [isCameraOn, setIsCameraOn] = useState(false);
+  const [cameraFacing, setCameraFacing] = useState<CameraType>('front');
+
+  const chatIdRef = useRef(chatId);
+  const modelIdRef = useRef(modelId);
+  chatIdRef.current = chatId;
+  modelIdRef.current = modelId;
+  isCameraOnRef.current = isCameraOn;
+
   const handleChatCreated = (id: string): void => {
     if (isVisible) {
       setChatId(id);
@@ -54,17 +69,26 @@ export function VoiceModeModal({ onChatCreated, ref, ...props }: VoiceModeModalP
   const { sendMessage, isLoading: isSending } = useSendMessage({ chatData: chat });
   const { startChatCreation, isLoading: isCreating } = useCreateNewChat({ onSuccess: handleChatCreated });
 
+  const sendMessageRef = useRef(sendMessage);
+  const startChatCreationRef = useRef(startChatCreation);
+  sendMessageRef.current = sendMessage;
+  startChatCreationRef.current = startChatCreation;
+
   const { isTranscribing, startSpeechRecording, stopSpeechRecording, completeSpeechRecording, metering } =
     useDictateMode({
       updateIntervalMillis: 100,
       onCompleteRecording: (text: string) => {
+        const attachedImages = pendingImageRef.current ? [pendingImageRef.current] : undefined;
+        pendingImageRef.current = null;
+
         if (text.trim().length) {
-          if (chatId) {
-            sendMessage(text, modelId);
+          if (chatIdRef.current) {
+            sendMessageRef.current(text, modelIdRef.current, undefined, undefined, attachedImages);
           } else {
-            startChatCreation(text, modelId);
+            startChatCreationRef.current(text, modelIdRef.current, undefined, undefined, attachedImages);
           }
 
+          speechStreamingService.resumeContentSpeaking();
           setIsWaitingNewMessage(true);
         } else {
           startSpeechRecording();
@@ -76,15 +100,24 @@ export function VoiceModeModal({ onChatCreated, ref, ...props }: VoiceModeModalP
   const isThinking =
     isCreating || isSending || isLoading || isTranscribing || isWaitingNewMessage || isReceivingNewMessage;
 
+  const stopCamera = (): void => {
+    setIsCameraOn(false);
+  };
+
   const close = async (): Promise<void> => {
-    await stopSpeechRecording();
-    await speechStreamingService.stopContentSpeaking();
+    // NOTE: Stop TTS immediately; isStopped is set sync so late handleContent/speakText no-ops
+    const stopSpeakingPromise = speechStreamingService.stopContentSpeaking();
+    speechStreamingService.clearListeners();
     clearSilenceTimeout();
+    stopCamera();
+    pendingImageRef.current = null;
     setIsUserSpeaking(false);
     setIsAiSpeaking(false);
     setIsWaitingNewMessage(false);
     setIsReceivingNewMessage(false);
     setIsVisible(false);
+    await stopSpeakingPromise;
+    await stopSpeechRecording();
   };
 
   useImperativeHandle(
@@ -102,7 +135,31 @@ export function VoiceModeModal({ onChatCreated, ref, ...props }: VoiceModeModalP
     [],
   );
 
-  const showUnderConstruction = (): void => ToastService.showFeatureNotImplemented();
+  const startCamera = async (): Promise<void> => {
+    const permission = await requestCameraPermission();
+
+    if (!permission.granted) {
+      ToastService.showError(i18n.t('SHARED.IMAGE_PICKER_SERVICE.TEXT_ACCESS_DENIED'));
+
+      return;
+    }
+
+    setIsCameraOn(true);
+  };
+
+  const flipCameraFacing = (): void => {
+    setCameraFacing((current) => (current === 'back' ? 'front' : 'back'));
+  };
+
+  const capturePendingImage = async (): Promise<void> => {
+    if (!isCameraOnRef.current) {
+      pendingImageRef.current = null;
+
+      return;
+    }
+
+    pendingImageRef.current = (await cameraPreviewRef.current?.takePicture()) ?? null;
+  };
 
   const clearSilenceTimeout = (): void => {
     if (silenceTimeout.current) {
@@ -117,8 +174,11 @@ export function VoiceModeModal({ onChatCreated, ref, ...props }: VoiceModeModalP
     }
 
     silenceTimeout.current = setTimeout(() => {
-      setIsUserSpeaking(false);
-      completeSpeechRecording();
+      void (async () => {
+        setIsUserSpeaking(false);
+        await capturePendingImage();
+        await completeSpeechRecording();
+      })();
     }, meteringSilenceDuration);
   };
 
@@ -137,20 +197,28 @@ export function VoiceModeModal({ onChatCreated, ref, ...props }: VoiceModeModalP
   }, [isVisible]);
 
   useEffect(() => {
-    if (isVisible) {
-      if (isWaitingNewMessage && newMessage && !newMessage.done) {
-        // NOTE: In this case, we start receiving a new message via WebSocket
+    if (!isVisible) {
+      return;
+    }
+
+    if (isWaitingNewMessage && newMessage) {
+      if (!newMessage.done) {
+        // NOTE: Start receiving a new message via WebSocket
         setIsWaitingNewMessage(false);
         setIsReceivingNewMessage(true);
         speechStreamingService.handleContent(newMessage.content);
+      } else if (newMessage.content.trim()) {
+        // NOTE: Reply already finished before streaming subscription (common on create-chat)
+        setIsWaitingNewMessage(false);
+        speechStreamingService.handleContent(newMessage.content, true);
       }
+    }
 
-      if (isReceivingNewMessage && newMessage) {
-        speechStreamingService.handleContent(newMessage.content, newMessage.done);
+    if (isReceivingNewMessage && newMessage) {
+      speechStreamingService.handleContent(newMessage.content, newMessage.done);
 
-        if (newMessage.done) {
-          setIsReceivingNewMessage(false);
-        }
+      if (newMessage.done) {
+        setIsReceivingNewMessage(false);
       }
     }
   }, [isVisible, isWaitingNewMessage, isReceivingNewMessage, newMessage?.content.length, newMessage?.done]);
@@ -188,13 +256,29 @@ export function VoiceModeModal({ onChatCreated, ref, ...props }: VoiceModeModalP
       {...props}>
       <View className='flex-1 bg-background-primary'>
         <AppSafeAreaView edges={['bottom']} className='flex-1'>
-          <View className='flex-1 items-center justify-center'>
-            {isThinking || isAiSpeaking ? <Loader /> : <SpeechListener metering={metering} />}
+          <View className='flex-1 items-center justify-center px-24'>
+            {isCameraOn ? (
+              <View className='w-full items-center justify-center'>
+                <CameraPreview
+                  ref={cameraPreviewRef}
+                  facing={cameraFacing}
+                  onClose={stopCamera} />
+                {(isThinking || isAiSpeaking) && (
+                  <View className='absolute inset-0 items-center justify-center'>
+                    <Loader />
+                  </View>
+                )}
+              </View>
+            ) : isThinking || isAiSpeaking ? (
+              <Loader />
+            ) : (
+              <SpeechListener metering={metering} />
+            )}
           </View>
           <View className='flex-row justify-between items-center p-24'>
             <IconButton
-              iconName='camera'
-              onPress={showUnderConstruction}
+              iconName={isCameraOn ? 'refresh' : 'camera'}
+              onPress={isCameraOn ? flipCameraFacing : startCamera}
               className='w-40 h-40 bg-background-secondary rounded-full'
             />
             <AppText className='text-sm-sm sm:text-sm'>

libs/mobile/chat/features/voice-mode-modal/src/lib/components/camera-preview/component.tsx

@@ -0,0 +1,73 @@
+import { CameraType, CameraView } from 'expo-camera';
+import { ForwardedRef, ReactElement, useImperativeHandle, useRef, useState } from 'react';
+import { StyleSheet } from 'react-native';
+import { IconButton, View } from '@open-webui-react-native/mobile/shared/ui/ui-kit';
+import { ImageData as ChatImageData } from '@open-webui-react-native/shared/data-access/common';
+
+export type CameraPreviewMethods = {
+  takePicture: () => Promise<ChatImageData | null>;
+};
+
+export type CameraPreviewRef = ForwardedRef<CameraPreviewMethods>;
+
+export interface CameraPreviewProps {
+  facing: CameraType;
+  onClose: () => void;
+  ref?: CameraPreviewRef;
+}
+
+const PICTURE_QUALITY = 0.2;
+
+export function CameraPreview({ facing, onClose, ref }: CameraPreviewProps): ReactElement {
+  const cameraRef = useRef<CameraView>(null);
+  const [isReady, setIsReady] = useState(false);
+
+  useImperativeHandle(
+    ref,
+    () => ({
+      takePicture: async (): Promise<ChatImageData | null> => {
+        if (!isReady || !cameraRef.current) {
+          return null;
+        }
+
+        try {
+          const photo = await cameraRef.current.takePictureAsync({
+            base64: true,
+            quality: PICTURE_QUALITY,
+            shutterSound: false,
+          });
+
+          if (!photo?.uri || !photo.base64) {
+            return null;
+          }
+
+          return {
+            uri: photo.uri,
+            base64: photo.base64,
+            mimeType: 'image/jpeg',
+          };
+        } catch {
+          return null;
+        }
+      },
+    }),
+    [isReady],
+  );
+
+  return (
+    <View className='w-full max-w-[420px] aspect-[3/4] rounded-3xl overflow-hidden bg-background-secondary'>
+      <CameraView
+        ref={cameraRef}
+        facing={facing}
+        mode='picture'
+        style={StyleSheet.absoluteFill}
+        onCameraReady={() => setIsReady(true)}
+      />
+      <IconButton
+        iconName='close'
+        onPress={onClose}
+        className='absolute top-16 right-16 w-40 h-40 bg-background-secondary/80 rounded-full'
+      />
+    </View>
+  );
+}

libs/mobile/chat/features/voice-mode-modal/src/lib/components/camera-preview/index.ts

@@ -0,0 +1 @@
+export * from './component';

libs/mobile/chat/features/voice-mode-modal/src/lib/components/index.ts

@@ -1,2 +1,3 @@
+export * from './camera-preview';
 export * from './loader';
 export * from './speech-listener';

libs/mobile/chat/features/voice-mode-modal/src/lib/config.ts

@@ -1,4 +1,4 @@
 export const voiceModeModalConfig = {
-  meteringSilenceThreshold: 0.3,
-  meteringSilenceDuration: 2500,
+  meteringSilenceThreshold: 0.5,
+  meteringSilenceDuration: 1500,
 };