microsoft · RushikeshGavali · Dec 26, 2024 · Dec 27, 2024 · Dec 27, 2024 · Dec 30, 2024
@@ -72,6 +72,7 @@ Notes: web developers are advised to use [`~` (tilde range)](https://github.com/
    -  Copy button is added to fenced code blocks (`<pre><code>`)
    -  Configure HTML sanitizer via `request.allowedTags`
 -  Added support for math blocks using `$$` delimiter alongside existing `\[...\]` and `\(...\)` notations, in PR [#5381](https://github.com/microsoft/BotFramework-WebChat/pull/5381), by [@OEvgeny](https://github.com/OEvgeny)
+- Introduced continuous listening capability, in PR [#5397](https://github.com/microsoft/BotFramework-WebChat/pull/5397), by [@RushikeshGavali](https://github.com/RushikeshGavali)
 
 ### Changed
 

diff --git a/__tests__/html2/continuousListening/continuousListening.html b/__tests__/html2/continuousListening/continuousListening.html
@@ -0,0 +1,102 @@
+<!DOCTYPE html>
+<html lang="en-US">
+  <head>
+    <link href="/assets/index.css" rel="stylesheet" type="text/css" />
+    <script crossorigin="anonymous" src="/test-harness.js"></script>
+    <script crossorigin="anonymous" src="/test-page-object.js"></script>
+    <script crossorigin="anonymous" src="/__dist__/webchat-es5.js"></script>
+  </head>
+  <body>
+    <main id="webchat"></main>
+    <script>
+      const {
+        testHelpers: {
+          speech: {
+            createQueuedArrayBufferAudioSource,
+            fetchSpeechData,
+          },
+        }
+      } = window;
+      run(async function () {
+        const { authorizationToken, region } = await fetch(
+          'https://hawo-mockbot4-token-app.blueriver-ce85e8f0.westus.azurecontainerapps.io/api/token/speech',
+          {
+            method: 'POST'
+          }
+        ).then(res => res.json());
+
+        const credentials = {
+          authorizationToken,
+          region
+        }
+
+        const audioConfig = createQueuedArrayBufferAudioSource();
+        const webSpeechPonyfillFactory = WebChat.createCognitiveServicesSpeechServicesPonyfillFactory({
+          audioConfig,
+          credentials
+        });
+
+        WebChat.renderWebChat(
+          {
+            directLine: WebChat.createDirectLine({ token: await testHelpers.token.fetchDirectLineToken() }),
+            store: testHelpers.createStore(),
+            enableContinuousListening: true,
+            webSpeechPonyfillFactory: () => {
+              return {
+                ...webSpeechPonyfillFactory(),
+              };
+            }
+          },
+          document.getElementById('webchat')
+        );
+
+        await pageConditions.uiConnected();
+
+        audioConfig.push(
+          await fetchSpeechData({
+            fetchCredentials: () => credentials,
+            text: 'Hello world'
+          })
+        );
+
+        audioConfig.push(
+          await fetchSpeechData({
+            fetchCredentials: () => credentials,
+            text: 'Continuous Listening'
+          })
+        );
+
+        // WHEN: When the microphone button is clicked, it should send "Hello world" and "Continuous Listening".
+        await host.click(pageElements.microphoneButton());
+
+        // wait until both messages are sent
+        await pageConditions.minNumActivitiesShown(3);
+
+        // THEN: Both messages should be sent.
+        await pageConditions.became(
+          'Recognize and send "Hello world"',
+          () =>
+            /hello\sworld/iu.test(
+              pageElements.activities()[0]?.querySelector('[aria-roledescription="message"]')?.innerText || ''
+            ),
+          5000
+        );
+
+        await pageConditions.became(
+          'Recognize and send "Continuous Listening"',
+          () =>
+            /Continuous\sListening/iu.test(
+              pageElements.activities()[2]?.querySelector('[aria-roledescription="message"]')?.innerText || ''
+            ),
+          5000
+        );
+        await host.click(pageElements.microphoneButton());
+
+        // THEN: The bot should respond to both messages.
+        await pageConditions.numActivitiesShown(4);
+
+        await host.snapshot('local');
+      });
+    </script>
+  </body>
+</html>
@@ -12,6 +12,7 @@ import {
   sendMessage,
   sendMessageBack,
   sendPostBack,
+  setContinuousListening,
   setDictateInterims,
   setDictateState,
   setLanguage,
@@ -263,6 +264,7 @@ type ComposerCoreProps = Readonly<{
   uiState?: 'blueprint' | 'disabled' | undefined;
   userID?: string;
   username?: string;
+  enableContinuousListening?: boolean;
 }>;
 
 const ComposerCore = ({
@@ -277,6 +279,7 @@ const ComposerCore = ({
   directLine,
   disabled,
   downscaleImageToDataURL,
+  enableContinuousListening,
   grammars,
   groupActivitiesMiddleware,
   internalErrorBoxClass,
@@ -311,7 +314,8 @@ const ComposerCore = ({
 
   useEffect(() => {
     dispatch(setLanguage(locale));
-  }, [dispatch, locale]);
+    dispatch(setContinuousListening(enableContinuousListening ?? false));
+  }, [dispatch, locale, enableContinuousListening]);
 
   useEffect(() => {
     dispatch(setSendTypingIndicator(!!sendTypingIndicator));

@@ -70,6 +70,7 @@ import useUIState from './useUIState';
 import useUserID from './useUserID';
 import useUsername from './useUsername';
 import useVoiceSelector from './useVoiceSelector';
+import useContinuousListening from './useContinuousListening';
 
 export {
   useActiveTyping,
@@ -143,5 +144,6 @@ export {
   useUIState,
   useUserID,
   useUsername,
-  useVoiceSelector
+  useVoiceSelector,
+  useContinuousListening
 };
@@ -0,0 +1,5 @@
+import { useSelector } from './internal/WebChatReduxContext';
+
+export default function useContinuousListening(): boolean {
+  return useSelector(({ continuousListening }) => continuousListening);
+}
@@ -21,7 +21,8 @@ const {
   useShouldSpeakIncomingActivity,
   useStopDictate,
   useSubmitSendBox,
-  useUIState
+  useUIState,
+  useContinuousListening
 } = hooks;
 
 const {
@@ -44,6 +45,7 @@ const Dictation = ({ onError }) => {
   const setDictateState = useSetDictateState();
   const stopDictate = useStopDictate();
   const submitSendBox = useSubmitSendBox();
+  const continuousListening = useContinuousListening();
 
   const numSpeakingActivities = useMemo(
     () => activities.filter(({ channelData: { speak } = {} }) => speak).length,
@@ -54,8 +56,10 @@ const Dictation = ({ onError }) => {
     ({ result: { confidence, transcript } = {} }) => {
       if (dictateState === DICTATING || dictateState === STARTING) {
         setDictateInterims([]);
-        setDictateState(IDLE);
-        stopDictate();
+        if (!continuousListening) {
+          setDictateState(IDLE);
+          stopDictate();
+        }
 
         if (transcript) {
           setSendBox(transcript);
@@ -65,6 +69,7 @@ const Dictation = ({ onError }) => {
       }
     },
     [
+      continuousListening,
       dictateState,
       setDictateInterims,
       setDictateState,

@@ -2,6 +2,8 @@
 /* eslint react/forbid-dom-props: "off" */
 
 import { hooks } from 'botframework-webchat-api';
+import { useSetDictateState } from 'botframework-webchat-api/internal';
+
 import { Constants } from 'botframework-webchat-core';
 import classNames from 'classnames';
 import memoize from 'memoize-one';
@@ -25,7 +27,8 @@ const {
   useShouldSpeakIncomingActivity,
   useStartDictate,
   useStopDictate,
-  useUIState
+  useUIState,
+  useContinuousListening
 } = hooks;
 
 const ROOT_STYLE = {
@@ -53,6 +56,8 @@ function useMicrophoneButtonClick(): () => void {
   const [webSpeechPonyfill] = useWebSpeechPonyfill();
   const startDictate = useStartDictate();
   const stopDictate = useStopDictate();
+  const setDictateState = useSetDictateState();
+  const continuousListening = useContinuousListening();
 
   const { speechSynthesis, SpeechSynthesisUtterance } = webSpeechPonyfill || {};
 
@@ -75,6 +80,9 @@ function useMicrophoneButtonClick(): () => void {
     } else if (dictateState === DictateState.DICTATING) {
       stopDictate();
       setSendBox(dictateInterims.join(' '));
+      if (continuousListening) {
+        setDictateState(DictateState.IDLE);
+      }
     } else {
       setShouldSpeakIncomingActivity(false);
       startDictate();
@@ -86,6 +94,8 @@ function useMicrophoneButtonClick(): () => void {
     dictateState,
     primeSpeechSynthesis,
     setSendBox,
+    setDictateState,
+    continuousListening,
     setShouldSpeakIncomingActivity,
     speechSynthesis,
     SpeechSynthesisUtterance,

@@ -0,0 +1,10 @@
+const SET_CONTINUOUS_LISTENING = 'WEB_CHAT/SET_CONTINUOUS_LISTENING';
+
+export default function setContinuousListening(continuousListening) {
+  return {
+    type: SET_CONTINUOUS_LISTENING,
+    payload: { continuousListening }
+  };
+}
+
+export { SET_CONTINUOUS_LISTENING };
@@ -1,6 +1,7 @@
 import { combineReducers } from 'redux';
 
 import connectivityStatus from './reducers/connectivityStatus';
+import continuousListening from './reducers/continuousListening';
 import createActivitiesReducer from './reducers/createActivitiesReducer';
 import createInternalReducer from './reducers/createInternalReducer';
 import createNotificationsReducer from './reducers/createNotificationsReducer';
@@ -38,6 +39,7 @@ export default function createReducer(ponyfill: GlobalScopePonyfill) {
     shouldSpeakIncomingActivity,
     suggestedActions,
     suggestedActionsOriginActivity,
-    typing: createTypingReducer(ponyfill)
+    typing: createTypingReducer(ponyfill),
+    continuousListening
   });
 }
@@ -12,6 +12,7 @@
 import sendMessage from './actions/sendMessage';
 import sendMessageBack from './actions/sendMessageBack';
 import sendPostBack from './actions/sendPostBack';
+import setContinuousListening from './actions/setContinuousListening'
 import setDictateInterims from './actions/setDictateInterims';
 import setDictateState from './actions/setDictateState';
 import setLanguage from './actions/setLanguage';
@@ -107,6 +108,7 @@
   sendMessage,
   sendMessageBack,
   sendPostBack,
+  setContinuousListening,
   setDictateInterims,
   setDictateState,
   setLanguage,

@@ -0,0 +1,16 @@
+import { SET_CONTINUOUS_LISTENING } from '../actions/setContinuousListening';
+
+const DEFAULT_STATE = false;
+
+export default function continuousListening(state = DEFAULT_STATE, { payload, type }) {
+  switch (type) {
+    case SET_CONTINUOUS_LISTENING:
+      state = payload.continuousListening;
+      break;
+
+    default:
+      break;
+  }
+
+  return state;
+}
@@ -1,8 +1,9 @@
-import { put, takeEvery } from 'redux-saga/effects';
+import { put, takeEvery, select } from 'redux-saga/effects';
 
 import { POST_ACTIVITY_PENDING } from '../actions/postActivity';
 import stopDictate from '../actions/stopDictate';
 import whileConnected from './effects/whileConnected';
+import continuousListeningSelector from '../selectors/continuousListening';
 
 function* stopDictateOnCardAction() {
   // TODO: [P2] We should stop speech input when the user click on anything on a card, including open URL which doesn't generate postActivity
@@ -14,7 +15,10 @@ function* stopDictateOnCardAction() {
     // In the future, if we have an action for card input, we should use that instead
     ({ payload, type }) => type === POST_ACTIVITY_PENDING && payload.activity.type === 'message',
     function* putStopDictate() {
-      yield put(stopDictate());
+      const continuousListening = yield select(continuousListeningSelector);
+      if (!continuousListening) {
+        yield put(stopDictate());
+      }
     }
   );
 }

@@ -0,0 +1,3 @@
+import type { ReduxState } from '../types/internal/ReduxState';
+
+export default ({ continuousListening }: ReduxState): boolean => continuousListening;
@@ -14,6 +14,7 @@ type ReduxState = {
   sendTimeout: number;
   sendTypingIndicator: boolean;
   shouldSpeakIncomingActivity: boolean;
+  continuousListening: boolean;
 };
 
 export type { ReduxState };
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,3 @@
		import type { ReduxState } from '../types/internal/ReduxState';

		export default ({ continuousListening }: ReduxState): boolean => continuousListening;