diff --git a/UnityProjects/MRTKDevTemplate/Packages/packages-lock.json b/UnityProjects/MRTKDevTemplate/Packages/packages-lock.json index 7e7616c8c..aa5fd09b0 100644 --- a/UnityProjects/MRTKDevTemplate/Packages/packages-lock.json +++ b/UnityProjects/MRTKDevTemplate/Packages/packages-lock.json @@ -428,7 +428,7 @@ "source": "local", "dependencies": { "com.microsoft.mrtk.graphicstools.unity": "0.5.12", - "org.mixedrealitytoolkit.core": "3.2.0", + "org.mixedrealitytoolkit.core": "3.3.0", "com.unity.inputsystem": "1.6.1", "com.unity.textmeshpro": "3.0.6", "com.unity.xr.interaction.toolkit": "2.3.0" diff --git a/org.mixedrealitytoolkit.core/CHANGELOG.md b/org.mixedrealitytoolkit.core/CHANGELOG.md index 6ac09fd09..0d21382ea 100644 --- a/org.mixedrealitytoolkit.core/CHANGELOG.md +++ b/org.mixedrealitytoolkit.core/CHANGELOG.md @@ -4,6 +4,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ## [3.3.0-development] - 2024-06-24 +### Added + +* Added event `OnSpeechRecognitionKeywordChanged` to allow UI updates when the speech recognition keyword has changed. [PR #792](https://github.com/MixedRealityToolkit/MixedRealityToolkit-Unity/pull/792/) + ### Fixed * Fixed broken project validation help link, for item 'MRTK3 profile may need to be assigned for the Standalone build target' (Issue #882) [PR#886 (https://github.com/MixedRealityToolkit/MixedRealityToolkit-Unity/pull/886)] diff --git a/org.mixedrealitytoolkit.core/Editor/Editors/StatefulInteractableEditor.cs b/org.mixedrealitytoolkit.core/Editor/Editors/StatefulInteractableEditor.cs index a2e079713..61f2509d3 100644 --- a/org.mixedrealitytoolkit.core/Editor/Editors/StatefulInteractableEditor.cs +++ b/org.mixedrealitytoolkit.core/Editor/Editors/StatefulInteractableEditor.cs @@ -22,6 +22,7 @@ public class StatefulInteractableEditor : BaseInteractableEditor private SerializedProperty allowSelectByVoice; private SerializedProperty SelectRequiresHover; private SerializedProperty speechRecognitionKeyword; + private SerializedProperty OnSpeechRecognitionKeywordChanged; private SerializedProperty VoiceRequiresFocus; private SerializedProperty UseGazeDwell; private SerializedProperty GazeDwellTime; @@ -33,6 +34,7 @@ public class StatefulInteractableEditor : BaseInteractableEditor private SerializedProperty OnEnabled; private SerializedProperty OnDisabled; private static bool advancedFoldout = false; + private static bool speechRecognitionKeywordEventFoldout = false; private static bool enabledEventsFoldout = false; /// @@ -53,6 +55,7 @@ protected override void OnEnable() allowSelectByVoice = SetUpProperty(nameof(allowSelectByVoice)); speechRecognitionKeyword = SetUpProperty(nameof(speechRecognitionKeyword)); + OnSpeechRecognitionKeywordChanged = SetUpAutoProperty(nameof(OnSpeechRecognitionKeywordChanged)); VoiceRequiresFocus = SetUpAutoProperty(nameof(VoiceRequiresFocus)); SelectRequiresHover = SetUpAutoProperty(nameof(SelectRequiresHover)); @@ -165,8 +168,13 @@ protected void DrawProperties(bool showToggleMode) { using (new EditorGUI.IndentLevelScope()) { - EditorGUILayout.PropertyField(speechRecognitionKeyword); EditorGUILayout.PropertyField(VoiceRequiresFocus); + EditorGUILayout.PropertyField(speechRecognitionKeyword); + speechRecognitionKeywordEventFoldout = EditorGUILayout.Foldout(speechRecognitionKeywordEventFoldout, EditorGUIUtility.TrTempContent("Speech Recognition Keyword event"), true); + if (speechRecognitionKeywordEventFoldout) + { + EditorGUILayout.PropertyField(OnSpeechRecognitionKeywordChanged); + } } } diff --git a/org.mixedrealitytoolkit.core/Interactables/StatefulInteractable.cs b/org.mixedrealitytoolkit.core/Interactables/StatefulInteractable.cs index 1b1724fbd..ab5bbdb7b 100644 --- a/org.mixedrealitytoolkit.core/Interactables/StatefulInteractable.cs +++ b/org.mixedrealitytoolkit.core/Interactables/StatefulInteractable.cs @@ -148,10 +148,17 @@ public string SpeechRecognitionKeyword { interactionManager.RegisterInteractable(this as IXRInteractable); } + OnSpeechRecognitionKeywordChanged.Invoke(speechRecognitionKeyword); } } } - + + /// + /// Fired when the has changed. + /// + [field: SerializeField, Tooltip("Fired when the Speech Recognition Keyword has changed.")] + public UnityEvent OnSpeechRecognitionKeywordChanged { get; private set; } = new UnityEvent(); + /// /// Does the voice command require this to have focus? /// If true, then the voice command will only respond to voice commands while this Interactable has focus. diff --git a/org.mixedrealitytoolkit.core/package.json b/org.mixedrealitytoolkit.core/package.json index fec5089e1..514d706d5 100644 --- a/org.mixedrealitytoolkit.core/package.json +++ b/org.mixedrealitytoolkit.core/package.json @@ -1,6 +1,6 @@ { "name": "org.mixedrealitytoolkit.core", - "version": "3.2.2-development", + "version": "3.3.0-development", "description": "A limited collection of common interfaces and utilities that most MRTK packages share. Most implementations of these interfaces are contained in other packages in the MRTK ecosystem.", "displayName": "MRTK Core Definitions", "msftFeatureCategory": "MRTK3", diff --git a/org.mixedrealitytoolkit.uxcore/CHANGELOG.md b/org.mixedrealitytoolkit.uxcore/CHANGELOG.md index b50d5616d..4ee93a810 100644 --- a/org.mixedrealitytoolkit.uxcore/CHANGELOG.md +++ b/org.mixedrealitytoolkit.uxcore/CHANGELOG.md @@ -2,7 +2,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). -## [3.2.1-development] - 2024-04-23 + +## [3.3.0-development] - 2024-06-24 + +### Added + +* Added automatic update for the `See It Say It Label` when the `SpeechRecognitionKeyword` of a `StatefulInteractable` has changed. Added ability to change the pattern, from inspector or code. When Unity Localization package is installed, a `LocalizedString` is used as pattern. [PR #792](https://github.com/MixedRealityToolkit/MixedRealityToolkit-Unity/pull/792) + +## [3.2.1] - 2024-04-23 ### Fixed diff --git a/org.mixedrealitytoolkit.uxcore/MRTK.UXCore.asmdef b/org.mixedrealitytoolkit.uxcore/MRTK.UXCore.asmdef index d24953a09..255905541 100644 --- a/org.mixedrealitytoolkit.uxcore/MRTK.UXCore.asmdef +++ b/org.mixedrealitytoolkit.uxcore/MRTK.UXCore.asmdef @@ -6,6 +6,7 @@ "MixedReality.Toolkit.Data", "Microsoft.MixedReality.GraphicsTools", "Unity.InputSystem", + "Unity.Localization", "Unity.TextMeshPro", "Unity.XR.Interaction.Toolkit", "Unity.XR.CoreUtils", @@ -39,6 +40,11 @@ "name": "org.mixedrealitytoolkit.windowsspeech", "expression": "", "define": "MRTK_SPEECH_PRESENT" + }, + { + "name": "com.unity.localization", + "expression": "", + "define": "UNITY_LOCALIZATION_PRESENT" } ], "noEngineReferences": false diff --git a/org.mixedrealitytoolkit.uxcore/SeeItSayIt/SeeItSayItLabelEnabler.cs b/org.mixedrealitytoolkit.uxcore/SeeItSayIt/SeeItSayItLabelEnabler.cs index cd3d62615..ffa8321ff 100644 --- a/org.mixedrealitytoolkit.uxcore/SeeItSayIt/SeeItSayItLabelEnabler.cs +++ b/org.mixedrealitytoolkit.uxcore/SeeItSayIt/SeeItSayItLabelEnabler.cs @@ -6,6 +6,9 @@ #if MRTK_INPUT_PRESENT && MRTK_SPEECH_PRESENT using MixedReality.Toolkit.Input; #endif +#if UNITY_LOCALIZATION_PRESENT +using UnityEngine.Localization; +#endif namespace MixedReality.Toolkit.UX { @@ -25,6 +28,16 @@ namespace MixedReality.Toolkit.UX [AddComponentMenu("MRTK/UX/See It Say It Label")] public class SeeItSayItLabelEnabler : MonoBehaviour { + /// + /// The present on the same GameObject. + /// + private PressableButton pressableButton; + + /// + /// The used to display the label present in child. + /// + private TMP_Text labelText; + [SerializeField] [Tooltip("The GameObject for the see-it say-it label to be enabled.")] private GameObject seeItSayItLabel; @@ -38,6 +51,32 @@ public GameObject SeeItSayItLabel set => seeItSayItLabel = value; } +#if UNITY_LOCALIZATION_PRESENT + [SerializeField] + [Tooltip("The LocalizedString that define the label pattern. Use a smart string with one argument that will be replaced by the button's speech recognition keyword (e.g: \"Say '{0}'\").")] + private LocalizedString localizedPattern; +#else + [SerializeField] + [Tooltip("The patern for the see-it say-it label using string.Format()")] + private string pattern = "Say '{0}'"; + + /// + /// The patern for the see-it say-it label using string.Format() + /// + public string Pattern + { + get => pattern; + set + { + pattern = value; + if (pressableButton != null) + { + UpdateLabel(pressableButton.SpeechRecognitionKeyword); + } + } + } +#endif + [SerializeField] [Tooltip("The Transform that the label will be dynamically positioned off of. Empty by default. If positioning a Canvas label, this must be a RectTransform.")] private Transform positionControl; @@ -54,13 +93,17 @@ public Transform PositionControl private float canvasOffset = -10f; private float nonCanvasOffset = -.004f; + protected virtual void Awake() + { + pressableButton = GetComponent(); + } + /// /// A Unity event function that is called on the frame when a script is enabled just before any of the update methods are called the first time. - /// + /// protected virtual void Start() { // Check if voice commands are enabled for this button - PressableButton pressableButton = gameObject.GetComponent(); if (pressableButton != null && pressableButton.AllowSelectByVoice) { // Check if input and speech packages are present @@ -72,6 +115,8 @@ protected virtual void Start() } SeeItSayItLabel.SetActive(true); + labelText = SeeItSayItLabel.GetComponentInChildren(true); + pressableButton.OnSpeechRecognitionKeywordChanged.AddListener(UpdateLabel); // Children must be disabled so that they are not initially visible foreach (Transform child in SeeItSayItLabel.transform) @@ -80,15 +125,7 @@ protected virtual void Start() } // Set the label text to reflect the speech recognition keyword - string keyword = pressableButton.SpeechRecognitionKeyword; - if (keyword != null) - { - TMP_Text labelText = SeeItSayItLabel.GetComponentInChildren(true); - if (labelText != null) - { - labelText.text = $"Say '{keyword}'"; - } - } + UpdateLabel(pressableButton.SpeechRecognitionKeyword); // If a Transform is specified, use it to reposition the object dynamically if (positionControl != null) @@ -97,7 +134,7 @@ protected virtual void Start() RectTransform controlRectTransform = PositionControl.gameObject.GetComponent(); // If PositionControl is a RectTransform, reposition label relative to Canvas button - if (controlRectTransform != null && SeeItSayItLabel.transform.childCount > 0) + if (controlRectTransform != null && SeeItSayItLabel.transform.childCount > 0) { // The parent RectTransform used to center the label RectTransform canvasTransform = SeeItSayItLabel.GetComponent(); @@ -107,7 +144,7 @@ protected virtual void Start() if (labelTransform != null && canvasTransform != null) { - labelTransform.anchoredPosition3D = new Vector3(canvasTransform.rect.width / 2f, canvasTransform.rect.height / 2f + (controlRectTransform.rect.height / 2f * -1) + canvasOffset, canvasOffset); + labelTransform.anchoredPosition3D = new Vector3(canvasTransform.rect.width / 2f, canvasTransform.rect.height / 2f + (controlRectTransform.rect.height / 2f * -1) + canvasOffset, canvasOffset); } } else @@ -115,8 +152,57 @@ protected virtual void Start() SeeItSayItLabel.transform.localPosition = new Vector3(PositionControl.localPosition.x, (PositionControl.lossyScale.y / 2f * -1) + nonCanvasOffset, PositionControl.localPosition.z + nonCanvasOffset); } } + +#if UNITY_LOCALIZATION_PRESENT + if (!localizedPattern.IsEmpty) + { + localizedPattern.StringChanged += OnLocalizedPatternChanged; + } +#endif +#endif + } + } + + protected virtual void OnDestroy() + { +#if MRTK_INPUT_PRESENT && MRTK_SPEECH_PRESENT + if (pressableButton != null) + { + pressableButton.OnSpeechRecognitionKeywordChanged.RemoveListener(UpdateLabel); +#if UNITY_LOCALIZATION_PRESENT + if (!localizedPattern.IsEmpty) + { + localizedPattern.StringChanged -= OnLocalizedPatternChanged; + } +#endif + } +#endif + } + + protected virtual void UpdateLabel(string keyword) + { +#if MRTK_INPUT_PRESENT && MRTK_SPEECH_PRESENT + if (!string.IsNullOrEmpty(keyword) && labelText != null) + { +#if UNITY_LOCALIZATION_PRESENT + if (!localizedPattern.IsEmpty) + { + labelText.text = localizedPattern.GetLocalizedString(keyword); + } + else + { + labelText.text = $"Say '{keyword}'"; + } +#else + labelText.text = string.Format(pattern, keyword); #endif } +#endif + } + + protected virtual void OnLocalizedPatternChanged(string value) + { + UpdateLabel(pressableButton.SpeechRecognitionKeyword); } } } diff --git a/org.mixedrealitytoolkit.uxcore/Tests/Runtime/SeeItSayItLabelEnablerTests.cs b/org.mixedrealitytoolkit.uxcore/Tests/Runtime/SeeItSayItLabelEnablerTests.cs index 406a393d7..f92b9b7de 100644 --- a/org.mixedrealitytoolkit.uxcore/Tests/Runtime/SeeItSayItLabelEnablerTests.cs +++ b/org.mixedrealitytoolkit.uxcore/Tests/Runtime/SeeItSayItLabelEnablerTests.cs @@ -26,19 +26,19 @@ public class SeeItSayItLabelEnablerTests : BaseRuntimeInputTests [UnityTest] public IEnumerator TestEnableAndSetLabel() { + GameObject testButton = SetUpButton(true, Control.None); + Transform label = testButton.transform.GetChild(0); + #if MRTK_INPUT_PRESENT && MRTK_SPEECH_PRESENT SpeechInteractor interactor = FindObjectUtility.FindAnyObjectByType(true); interactor.gameObject.SetActive(true); - GameObject testButton = SetUpButton(true, Control.None); yield return null; if (Application.isBatchMode) { LogAssert.Expect(LogType.Exception, new Regex("Speech recognition is not supported on this machine")); } - Transform label = testButton.transform.GetChild(0); - Transform sublabel = label.transform.GetChild(0); Assert.IsTrue(label.gameObject.activeSelf, "Label is enabled"); Assert.IsTrue(!sublabel.gameObject.activeSelf, "Child objects are disabled"); @@ -53,6 +53,43 @@ public IEnumerator TestEnableAndSetLabel() yield return null; } + [UnityTest] + public IEnumerator TestAutoUpdateLabel() + { + GameObject testButton = SetUpButton(true, Control.None); + Transform label = testButton.transform.GetChild(0); + +#if MRTK_INPUT_PRESENT && MRTK_SPEECH_PRESENT + SpeechInteractor interactor = FindObjectUtility.FindAnyObjectByType(true); + interactor.gameObject.SetActive(true); + + yield return null; + if (Application.isBatchMode) + { + LogAssert.Expect(LogType.Exception, new Regex("Speech recognition is not supported on this machine")); + } + + Transform sublabel = label.transform.GetChild(0); + TMP_Text text = label.gameObject.GetComponentInChildren(true); + Assert.AreEqual(text.text, "Say 'test'", "Label text was set to voice command keyword."); + + testButton.GetComponent().SpeechRecognitionKeyword = "hello world"; + + Assert.AreEqual(text.text, "Say 'hello world'", "Label text was updated according to voice command keyword."); +#else + Assert.IsTrue(!label.gameObject.activeSelf, "Did not enable label because voice commands unavailable."); +#endif + + Object.Destroy(testButton); + // Wait for a frame to give Unity a change to actually destroy the object + yield return null; + // The speech recognition keyword change will trigger this exception at next update when speech recognition is not supported + if (Application.isBatchMode) + { + LogAssert.Expect(LogType.Exception, new Regex("Speech recognition is not supported on this machine")); + } + } + [UnityTest] public IEnumerator TestVoiceCommandsUnavailable() { @@ -70,18 +107,19 @@ public IEnumerator TestVoiceCommandsUnavailable() [UnityTest] public IEnumerator TestPositionCanvasLabel() { + GameObject testButton = SetUpButton(true, Control.Canvas); + Transform label = testButton.transform.GetChild(0); + #if MRTK_INPUT_PRESENT && MRTK_SPEECH_PRESENT SpeechInteractor interactor = FindObjectUtility.FindAnyObjectByType(true); interactor.gameObject.SetActive(true); - GameObject testButton = SetUpButton(true, Control.Canvas); yield return null; if (Application.isBatchMode) { LogAssert.Expect(LogType.Exception, new Regex("Speech recognition is not supported on this machine")); } - Transform label = testButton.transform.GetChild(0); RectTransform sublabel = label.transform.GetChild(0) as RectTransform; Assert.AreEqual(sublabel.anchoredPosition3D, new Vector3(10, -30, -10), "Label is positioned correctly"); #else @@ -96,18 +134,19 @@ public IEnumerator TestPositionCanvasLabel() [UnityTest] public IEnumerator TestPositionNonCanvasLabel() { + GameObject testButton = SetUpButton(true, Control.NonCanvas); + Transform label = testButton.transform.GetChild(0); + #if MRTK_INPUT_PRESENT && MRTK_SPEECH_PRESENT SpeechInteractor interactor = FindObjectUtility.FindAnyObjectByType(true); interactor.gameObject.SetActive(true); - GameObject testButton = SetUpButton(true, Control.NonCanvas); yield return null; if (Application.isBatchMode) { LogAssert.Expect(LogType.Exception, new Regex("Speech recognition is not supported on this machine")); } - Transform label = testButton.transform.GetChild(0); Assert.AreEqual(label.transform.localPosition, new Vector3(10f, -.504f, -.004f), "Label is positioned correctly"); #else Assert.IsTrue(!label.gameObject.activeSelf, "Did not enable label because voice commands unavailable."); diff --git a/org.mixedrealitytoolkit.uxcore/package.json b/org.mixedrealitytoolkit.uxcore/package.json index 01e6e2ccf..6640993e7 100644 --- a/org.mixedrealitytoolkit.uxcore/package.json +++ b/org.mixedrealitytoolkit.uxcore/package.json @@ -1,6 +1,6 @@ { "name": "org.mixedrealitytoolkit.uxcore", - "version": "3.2.1-development", + "version": "3.3.0-development", "description": "Core interaction and visualization scripts for building MR UI components. Intended to be consumed when building UX libraries. For pre-existing library of components see the UX Components package.", "displayName": "MRTK UX Core Scripts", "msftFeatureCategory": "MRTK3", @@ -18,7 +18,7 @@ "documentationUrl": "https://www.mixedrealitytoolkit.org", "dependencies": { "com.microsoft.mrtk.graphicstools.unity": "0.5.12", - "org.mixedrealitytoolkit.core": "3.2.0", + "org.mixedrealitytoolkit.core": "3.3.0", "com.unity.inputsystem": "1.6.1", "com.unity.textmeshpro": "3.0.6", "com.unity.xr.interaction.toolkit": "2.3.0"