Discussions

I’ve set task_type to REPEAT in voice chat mode and the avatar is responding back with a general knowledge answer (from its default settings I’m guessing) AND THEN it repeats back the text I sent it from Anthropic’s Claude. How do I get it to not respond with the first response from its general knowledge base?

Here is my code:

 const createNewSession = async (avatarId: string) => {  
    if (isCreatingSessionRef.current || sessionStatus === 'creating' || sessionStatus === 'started') {  
        console.warn('Session creation already in progress or started');  
        return;  
}
   
isCreatingSessionRef.current = true;
setSessionStatus('creating');

setIsAvatarLoading(true);

try {
  console.log('Creating a new session...');
  const accessToken = await getAccessToken();
  avatar.current = new StreamingAvatar({ token: accessToken });

  const sessionInfo = await avatar.current.createStartAvatar({
    quality: AvatarQuality.High,
    avatarName: avatarId,
    voice: { voiceId: voiceIDRef.current?.value || '' },
  });

  // Add event listeners after creating the avatar instance
  avatar.current.on(StreamingEvents.AVATAR_START_TALKING, (e) => {
    console.log('Avatar started talking', e);
  });

  avatar.current.on(StreamingEvents.AVATAR_STOP_TALKING, (e) => {
    console.log('Avatar stopped talking', e);
  });

  avatar.current.on(StreamingEvents.STREAM_DISCONNECTED, () => {
    console.log('Stream disconnected');
    handleStopSession();
  });

  avatar.current.on(StreamingEvents.USER_START, (event) => {
    console.log('User has started interaction:', event);
    // Handle the start of the user's interaction, such as activating a listening indicator
  });

  avatar.current.on(StreamingEvents.USER_STOP, (event) => {
    console.log('User has stopped interaction:', event);
    // Handle the end of the user's interaction, such as deactivating a listening indicator
  });

  // Add event listener for USER_TALKING_MESSAGE
  avatar.current.on(StreamingEvents.USER_TALKING_MESSAGE, handleUserTalkingMessage);

  avatar.current.on(StreamingEvents.USER_END_MESSAGE, (message) => {
    console.log('User end message:', message);
    // Handle the end of the user's message, e.g., process the user's response
  });

  avatar.current.on(StreamingEvents.STREAM_READY, async (event) => {
    console.log('Stream ready:', event.detail);
    if (videoRef.current) {
      videoRef.current.srcObject = event.detail;
      try {
        await videoRef.current.play();
        console.log('Video playback started successfully');
        setIsAvatarReady(true);
        setIsAvatarLoading(false); // Loader will go away when avatar is ready

        // Start voice chat mode here with enableAvatarResponse set to false
        await avatar.current!.startVoiceChat({ 
          useSilencePrompt: true, 
        });

      } catch (error) {
        console.error('Error playing video:', error);
        handleStatusUpdate('Failed to start video. Please try again.');
      }
    }
  });

  // Set the session ID
  if (!sessionInfo.sessionId) {
    throw new Error('Session ID not returned');
  }
  setSessionId(sessionInfo.sessionId);
  setIsSessionCreated(true);
  setSessionStatus('started');
  setIsVideoButtonVisible(true);
  handleStatusUpdate('Session created successfully. Waiting for stream to be ready...');

} catch (error) {
  console.error('Error creating session:', error);
  handleStatusUpdate('Failed to create session');
  setIsAvatarLoading(false); // Ensure loader is hidden on error
} finally {
  isCreatingSessionRef.current = false;
}
};

const handleUserTalkingMessage = (event: CustomEvent) => {
    const spokenMessage = event.detail.message;
    console.log('User spoken message:', spokenMessage);
    
    // Process the spoken message as if it was typed and sent
    sendMessage(spokenMessage);
  };

  const sendToAvatar = async (text: string): Promise<void> => {
    if (avatar.current) {
      try {
        await avatar.current.speak({ text: text, task_type: TaskType.REPEAT }).catch((e) => {
          
          
        });
        
      } catch (e) {
        console.error('Error sending task to avatar:', e);
        handleStatusUpdate('Failed to send task to avatar');
        throw e; // Rethrow to handle in processPhraseQueue
      }
    }
  };