Split hinter tokens at Unicode word boundaries

src/hinter/cwd_aware.rs

@@ -1,4 +1,5 @@
 use crate::{
+    hinter::get_first_token,
     history::SearchQuery,
     result::{ReedlineError, ReedlineErrorVariants::HistoryFeatureUnsupported},
     Hinter, History,
@@ -63,21 +64,7 @@ impl Hinter for CwdAwareHinter {
     }
 
     fn next_hint_token(&self) -> String {
-        let mut reached_content = false;
-        let result: String = self
-            .current_hint
-            .chars()
-            .take_while(|c| match (c.is_whitespace(), reached_content) {
-                (true, true) => false,
-                (true, false) => true,
-                (false, true) => true,
-                (false, false) => {
-                    reached_content = true;
-                    true
-                }
-            })
-            .collect();
-        result
+        get_first_token(&self.current_hint)
     }
 }
 

src/hinter/default.rs

@@ -1,4 +1,4 @@
-use crate::{history::SearchQuery, Hinter, History};
+use crate::{hinter::get_first_token, history::SearchQuery, Hinter, History};
 use nu_ansi_term::{Color, Style};
 
 /// A hinter that uses the completions or the history to show a hint to the user
@@ -47,21 +47,7 @@ impl Hinter for DefaultHinter {
     }
 
     fn next_hint_token(&self) -> String {
-        let mut reached_content = false;
-        let result: String = self
-            .current_hint
-            .chars()
-            .take_while(|c| match (c.is_whitespace(), reached_content) {
-                (true, true) => false,
-                (true, false) => true,
-                (false, true) => true,
-                (false, false) => {
-                    reached_content = true;
-                    true
-                }
-            })
-            .collect();
-        result
+        get_first_token(&self.current_hint)
     }
 }
 

src/hinter/mod.rs

@@ -3,6 +3,30 @@ mod default;
 pub use cwd_aware::CwdAwareHinter;
 pub use default::DefaultHinter;
 
+use unicode_segmentation::UnicodeSegmentation;
+
+pub fn is_whitespace_str(s: &str) -> bool {
+    s.chars().all(char::is_whitespace)
+}
+
+pub fn get_first_token(string: &str) -> String {
+    let mut reached_content = false;
+    let result = string
+        .split_word_bounds()
+        .take_while(|word| match (is_whitespace_str(word), reached_content) {
+            (_, true) => false,
+            (true, false) => true,
+            (false, false) => {
+                reached_content = true;
+                true
+            }
+        })
+        .collect::<Vec<&str>>()
+        .join("")
+        .to_string();
+    result
+}
+
 use crate::History;
 /// A trait that's responsible for returning the hint for the current line and position
 /// Hints are often shown in-line as part of the buffer, showing the user text they can accept or ignore