SeeItSayItLabel improvements (#593)

UnityProjects/MRTKDevTemplate/Packages/packages-lock.json

@@ -428,7 +428,7 @@
       "source": "local",
       "dependencies": {
         "com.microsoft.mrtk.graphicstools.unity": "0.5.12",
-        "org.mixedrealitytoolkit.core": "3.2.0",
+        "org.mixedrealitytoolkit.core": "3.3.0",
         "com.unity.inputsystem": "1.6.1",
         "com.unity.textmeshpro": "3.0.6",
         "com.unity.xr.interaction.toolkit": "2.3.0"

org.mixedrealitytoolkit.core/CHANGELOG.md

@@ -4,6 +4,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 
 ## [3.3.0-development] - 2024-06-24
 
+### Added
+
+* Added event `OnSpeechRecognitionKeywordChanged` to allow UI updates when the speech recognition keyword has changed. [PR #792](https://github.com/MixedRealityToolkit/MixedRealityToolkit-Unity/pull/792/)
+
 ### Fixed
 
 * Fixed broken project validation help link, for item 'MRTK3 profile may need to be assigned for the Standalone build target' (Issue #882) [PR#886 (https://github.com/MixedRealityToolkit/MixedRealityToolkit-Unity/pull/886)]

org.mixedrealitytoolkit.core/Editor/Editors/StatefulInteractableEditor.cs

@@ -22,6 +22,7 @@ public class StatefulInteractableEditor : BaseInteractableEditor
         private SerializedProperty allowSelectByVoice;
         private SerializedProperty SelectRequiresHover;
         private SerializedProperty speechRecognitionKeyword;
+        private SerializedProperty OnSpeechRecognitionKeywordChanged;
         private SerializedProperty VoiceRequiresFocus;
         private SerializedProperty UseGazeDwell;
         private SerializedProperty GazeDwellTime;
@@ -33,6 +34,7 @@ public class StatefulInteractableEditor : BaseInteractableEditor
         private SerializedProperty OnEnabled;
         private SerializedProperty OnDisabled;
         private static bool advancedFoldout = false;
+        private static bool speechRecognitionKeywordEventFoldout = false;
         private static bool enabledEventsFoldout = false;
 
         /// <summary>
@@ -53,6 +55,7 @@ protected override void OnEnable()
 
             allowSelectByVoice = SetUpProperty(nameof(allowSelectByVoice));
             speechRecognitionKeyword = SetUpProperty(nameof(speechRecognitionKeyword));
+            OnSpeechRecognitionKeywordChanged = SetUpAutoProperty(nameof(OnSpeechRecognitionKeywordChanged));
             VoiceRequiresFocus = SetUpAutoProperty(nameof(VoiceRequiresFocus));
 
             SelectRequiresHover = SetUpAutoProperty(nameof(SelectRequiresHover));
@@ -165,8 +168,13 @@ protected void DrawProperties(bool showToggleMode)
                     {
                         using (new EditorGUI.IndentLevelScope())
                         {
-                            EditorGUILayout.PropertyField(speechRecognitionKeyword);
                             EditorGUILayout.PropertyField(VoiceRequiresFocus);
+                            EditorGUILayout.PropertyField(speechRecognitionKeyword);
+                            speechRecognitionKeywordEventFoldout = EditorGUILayout.Foldout(speechRecognitionKeywordEventFoldout, EditorGUIUtility.TrTempContent("Speech Recognition Keyword event"), true);
+                            if (speechRecognitionKeywordEventFoldout)
+                            {
+                                EditorGUILayout.PropertyField(OnSpeechRecognitionKeywordChanged);
+                            }
                         }
                     }
 

org.mixedrealitytoolkit.core/Interactables/StatefulInteractable.cs

@@ -148,10 +148,17 @@ public string SpeechRecognitionKeyword
                     {
                         interactionManager.RegisterInteractable(this as IXRInteractable);
                     }
+                    OnSpeechRecognitionKeywordChanged.Invoke(speechRecognitionKeyword);
                 }
             }
         }
-
+
+        /// <summary>
+        /// Fired when the <see cref="SpeechRecognitionKeyword"/> has changed.
+        /// </summary>
+        [field: SerializeField, Tooltip("Fired when the Speech Recognition Keyword has changed.")]
+        public UnityEvent<string> OnSpeechRecognitionKeywordChanged { get; private set; } = new UnityEvent<string>();
+
         /// <summary>
         /// Does the voice command require this to have focus?
         /// If true, then the voice command will only respond to voice commands while this Interactable has focus.

org.mixedrealitytoolkit.core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "org.mixedrealitytoolkit.core",
-  "version": "3.2.2-development",
+  "version": "3.3.0-development",
   "description": "A limited collection of common interfaces and utilities that most MRTK packages share. Most implementations of these interfaces are contained in other packages in the MRTK ecosystem.",
   "displayName": "MRTK Core Definitions",
   "msftFeatureCategory": "MRTK3",

org.mixedrealitytoolkit.uxcore/CHANGELOG.md

@@ -2,7 +2,14 @@
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 
-## [3.2.1-development] - 2024-04-23
+
+## [3.3.0-development] - 2024-06-24
+
+### Added
+
+* Added automatic update for the `See It Say It Label` when the `SpeechRecognitionKeyword` of a `StatefulInteractable` has changed. Added ability to change the pattern, from inspector or code. When Unity Localization package is installed, a `LocalizedString` is used as pattern.  [PR #792](https://github.com/MixedRealityToolkit/MixedRealityToolkit-Unity/pull/792)
+
+## [3.2.1] - 2024-04-23
 
 ### Fixed
 

org.mixedrealitytoolkit.uxcore/MRTK.UXCore.asmdef

@@ -6,6 +6,7 @@
         "MixedReality.Toolkit.Data",
         "Microsoft.MixedReality.GraphicsTools",
         "Unity.InputSystem",
+        "Unity.Localization",
         "Unity.TextMeshPro",
         "Unity.XR.Interaction.Toolkit",
         "Unity.XR.CoreUtils",
@@ -39,6 +40,11 @@
             "name": "org.mixedrealitytoolkit.windowsspeech",
             "expression": "",
             "define": "MRTK_SPEECH_PRESENT"
+        },
+        {
+            "name": "com.unity.localization",
+            "expression": "",
+            "define": "UNITY_LOCALIZATION_PRESENT"
         }
     ],
     "noEngineReferences": false

org.mixedrealitytoolkit.uxcore/SeeItSayIt/SeeItSayItLabelEnabler.cs

@@ -6,6 +6,9 @@
 #if MRTK_INPUT_PRESENT && MRTK_SPEECH_PRESENT
 using MixedReality.Toolkit.Input;
 #endif
+#if UNITY_LOCALIZATION_PRESENT
+using UnityEngine.Localization;
+#endif
 
 namespace MixedReality.Toolkit.UX
 {
@@ -25,6 +28,16 @@ namespace MixedReality.Toolkit.UX
     [AddComponentMenu("MRTK/UX/See It Say It Label")]
     public class SeeItSayItLabelEnabler : MonoBehaviour
     {
+        /// <summary>
+        /// The <see cref="PressableButton"/> present on the same GameObject.
+        /// </summary>
+        private PressableButton pressableButton;
+
+        /// <summary>
+        /// The <see cref="TMP_Text"/> used to display the label present in child.
+        /// </summary>
+        private TMP_Text labelText;
+
         [SerializeField]
         [Tooltip("The GameObject for the see-it say-it label to be enabled.")]
         private GameObject seeItSayItLabel;
@@ -38,6 +51,32 @@ public GameObject SeeItSayItLabel
             set => seeItSayItLabel = value;
         }
 
+#if UNITY_LOCALIZATION_PRESENT
+        [SerializeField]
+        [Tooltip("The LocalizedString that define the label pattern. Use a smart string with one argument that will be replaced by the button's speech recognition keyword (e.g: \"Say '{0}'\").")]
+        private LocalizedString localizedPattern;
+#else
+        [SerializeField]
+        [Tooltip("The patern for the see-it say-it label using string.Format()")]
+        private string pattern = "Say '{0}'";
+
+        /// <summary>
+        /// The patern for the see-it say-it label using string.Format()
+        /// </summary>
+        public string Pattern
+        {
+            get => pattern;
+            set
+            {
+                pattern = value;
+                if (pressableButton != null)
+                {
+                    UpdateLabel(pressableButton.SpeechRecognitionKeyword);
+                }
+            }
+        }
+#endif
+
         [SerializeField]
         [Tooltip("The Transform that the label will be dynamically positioned off of. Empty by default. If positioning a Canvas label, this must be a RectTransform.")]
         private Transform positionControl;
@@ -54,13 +93,17 @@ public Transform PositionControl
         private float canvasOffset = -10f;
         private float nonCanvasOffset = -.004f;
 
+        protected virtual void Awake()
+        {
+            pressableButton = GetComponent<PressableButton>();
+        }
+
         /// <summary>
         /// A Unity event function that is called on the frame when a script is enabled just before any of the update methods are called the first time.
-        /// </summary> 
+        /// </summary>
         protected virtual void Start()
         {
             // Check if voice commands are enabled for this button
-            PressableButton pressableButton = gameObject.GetComponent<PressableButton>();
             if (pressableButton != null && pressableButton.AllowSelectByVoice)
             {
                 // Check if input and speech packages are present
@@ -72,6 +115,8 @@ protected virtual void Start()
                 }
 
                 SeeItSayItLabel.SetActive(true);
+                labelText = SeeItSayItLabel.GetComponentInChildren<TMP_Text>(true);
+                pressableButton.OnSpeechRecognitionKeywordChanged.AddListener(UpdateLabel);
 
                 // Children must be disabled so that they are not initially visible 
                 foreach (Transform child in SeeItSayItLabel.transform)
@@ -80,15 +125,7 @@ protected virtual void Start()
                 }
 
                 // Set the label text to reflect the speech recognition keyword
-                string keyword = pressableButton.SpeechRecognitionKeyword;
-                if (keyword != null)
-                {
-                    TMP_Text labelText = SeeItSayItLabel.GetComponentInChildren<TMP_Text>(true);
-                    if (labelText != null)
-                    {
-                        labelText.text = $"Say '{keyword}'";
-                    }
-                }
+                UpdateLabel(pressableButton.SpeechRecognitionKeyword);
 
                 // If a Transform is specified, use it to reposition the object dynamically
                 if (positionControl != null)
@@ -97,7 +134,7 @@ protected virtual void Start()
                     RectTransform controlRectTransform = PositionControl.gameObject.GetComponent<RectTransform>();
 
                     // If PositionControl is a RectTransform, reposition label relative to Canvas button
-                    if (controlRectTransform != null &&  SeeItSayItLabel.transform.childCount > 0)
+                    if (controlRectTransform != null && SeeItSayItLabel.transform.childCount > 0)
                     {
                         // The parent RectTransform used to center the label
                         RectTransform canvasTransform = SeeItSayItLabel.GetComponent<RectTransform>();
@@ -107,16 +144,65 @@ protected virtual void Start()
 
                         if (labelTransform != null && canvasTransform != null)
                         {
-                            labelTransform.anchoredPosition3D = new Vector3(canvasTransform.rect.width / 2f, canvasTransform.rect.height / 2f + (controlRectTransform.rect.height /  2f * -1) + canvasOffset, canvasOffset);
+                            labelTransform.anchoredPosition3D = new Vector3(canvasTransform.rect.width / 2f, canvasTransform.rect.height / 2f + (controlRectTransform.rect.height / 2f * -1) + canvasOffset, canvasOffset);
                         }
                     }
                     else
                     {
                         SeeItSayItLabel.transform.localPosition = new Vector3(PositionControl.localPosition.x, (PositionControl.lossyScale.y / 2f * -1) + nonCanvasOffset, PositionControl.localPosition.z + nonCanvasOffset);
                     }
                 }
+
+#if UNITY_LOCALIZATION_PRESENT
+                if (!localizedPattern.IsEmpty)
+                {
+                    localizedPattern.StringChanged += OnLocalizedPatternChanged;
+                }
+#endif
+#endif
+            }
+        }
+
+        protected virtual void OnDestroy()
+        {
+#if MRTK_INPUT_PRESENT && MRTK_SPEECH_PRESENT
+            if (pressableButton != null)
+            {
+                pressableButton.OnSpeechRecognitionKeywordChanged.RemoveListener(UpdateLabel);
+#if UNITY_LOCALIZATION_PRESENT
+                if (!localizedPattern.IsEmpty)
+                {
+                    localizedPattern.StringChanged += OnLocalizedPatternChanged;
+                }
+#endif
+            }
+#endif
+            }
+
+        protected virtual void UpdateLabel(string keyword)
+        {
+#if MRTK_INPUT_PRESENT && MRTK_SPEECH_PRESENT
+            if (!string.IsNullOrEmpty(keyword) && labelText != null)
+            {
+#if UNITY_LOCALIZATION_PRESENT
+                if (!localizedPattern.IsEmpty)
+                {
+                    labelText.text = localizedPattern.GetLocalizedString(keyword);
+                }
+                else
+                {
+                    labelText.text = $"Say '{keyword}'";
+                }
+#else
+                labelText.text = string.Format(pattern, keyword);
 #endif
             }
+#endif
+        }
+
+        protected virtual void OnLocalizedPatternChanged(string value)
+        {
+            UpdateLabel(pressableButton.SpeechRecognitionKeyword);
         }
     }
 }