Change ParseOutput to return a NonTerminal instead of a Node (#…

…1187)

Solves #1184
To go on top of #1172

This has a big impact in the failing examples, which now return the
expected `NonTerminal`.

This was acknowledgedly coded by playing the whack-a-mole with the type
system. I'm not so happy with the bunch of `Rc::clone()` that leaked
out; I will consider an alternative once we settle this is what we need
and there are no more pressing issues.

Currently doing a full CI run locally to find what more to fix. In the
meantime, I'll mark this as draft.

.changeset/young-bottles-beam.md

@@ -0,0 +1,5 @@
+---
+"@slang-private/codegen-runtime-npm-package": patch
+---
+
+Change `ParseOutput` and `File.tree` to return a `NonTerminal` instead of a `Node`

crates/codegen/runtime/cargo/crate/src/runtime/compilation/file.rs

@@ -1,19 +1,20 @@
 use std::collections::BTreeMap;
+use std::rc::Rc;
 
 use metaslang_cst::text_index::TextIndex;
 
-use crate::cst::{Cursor, Node};
+use crate::cst::{Cursor, NonterminalNode};
 
 #[derive(Clone)]
 pub struct File {
     id: String,
-    tree: Node,
+    tree: Rc<NonterminalNode>,
 
     resolved_imports: BTreeMap<usize, String>,
 }
 
 impl File {
-    pub(super) fn new(id: String, tree: Node) -> Self {
+    pub(super) fn new(id: String, tree: Rc<NonterminalNode>) -> Self {
         Self {
             id,
             tree,
@@ -26,12 +27,12 @@ impl File {
         &self.id
     }
 
-    pub fn tree(&self) -> &Node {
+    pub fn tree(&self) -> &Rc<NonterminalNode> {
         &self.tree
     }
 
     pub fn create_tree_cursor(&self) -> Cursor {
-        self.tree.clone().cursor_with_offset(TextIndex::ZERO)
+        Rc::clone(&self.tree).cursor_with_offset(TextIndex::ZERO)
     }
 
     pub(super) fn resolve_import(&mut self, import_path: &Cursor, destination_file_id: String) {

crates/codegen/runtime/cargo/crate/src/runtime/compilation/internal_builder.rs

@@ -43,7 +43,7 @@ impl InternalCompilationBuilder {
 
         let import_paths = self.imports.extract(parse_output.create_tree_cursor());
 
-        let file = File::new(id.clone(), parse_output.tree().clone());
+        let file = File::new(id.clone(), Rc::clone(parse_output.tree()));
         self.files.insert(id, file);
 
         AddFileResponse { import_paths }

crates/codegen/runtime/cargo/crate/src/runtime/parser/lexer/mod.rs

@@ -94,7 +94,7 @@ pub(crate) trait Lexer {
             .is_some_and(|t| t.accepted_as(kind))
         {
             input.set_position(start);
-            return ParserResult::no_match(None, vec![kind]);
+            return ParserResult::no_match(vec![kind]);
         }
         let end = input.position();
 
@@ -129,7 +129,7 @@ pub(crate) trait Lexer {
             .is_some_and(|t| t.accepted_as(kind))
         {
             input.set_position(restore);
-            return ParserResult::no_match(None, vec![kind]);
+            return ParserResult::no_match(vec![kind]);
         }
         let end = input.position();
         children.push(Edge::anonymous(Node::terminal(

crates/codegen/runtime/cargo/crate/src/runtime/parser/parse_output.rs

@@ -1,14 +1,16 @@
-use crate::cst::{Cursor, Node, TextIndex};
+use std::rc::Rc;
+
+use crate::cst::{Cursor, NonterminalNode, TextIndex};
 use crate::parser::ParseError;
 
 #[derive(Clone, Debug, PartialEq)]
 pub struct ParseOutput {
-    pub(crate) tree: Node,
+    pub(crate) tree: Rc<NonterminalNode>,
     pub(crate) errors: Vec<ParseError>,
 }
 
 impl ParseOutput {
-    pub fn tree(&self) -> &Node {
+    pub fn tree(&self) -> &Rc<NonterminalNode> {
         &self.tree
     }
 
@@ -22,6 +24,6 @@ impl ParseOutput {
 
     /// Creates a cursor that starts at the root of the parse tree.
     pub fn create_tree_cursor(&self) -> Cursor {
-        self.tree.clone().cursor_with_offset(TextIndex::ZERO)
+        Rc::clone(&self.tree).cursor_with_offset(TextIndex::ZERO)
     }
 }

crates/codegen/runtime/cargo/crate/src/runtime/parser/parser.rs.jinja2

@@ -108,7 +108,7 @@ impl Parser {
         {%- else -%}
             match kind {
                 {%- for parser_name, _ in model.parser.parser_functions -%}
-                    NonterminalKind::{{ parser_name }} => Self::{{ parser_name | snake_case }}.parse(self, input),
+                    NonterminalKind::{{ parser_name }} => Self::{{ parser_name | snake_case }}.parse(self, input, kind),
                 {%- endfor -%}
             }
         {%- endif -%}

crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/parser_function.rs

@@ -1,6 +1,8 @@
 use std::rc::Rc;
 
-use crate::cst::{Edge, Node, TerminalKind, TerminalKindExtensions, TextIndex};
+use crate::cst::{
+    Edge, Node, NonterminalKind, NonterminalNode, TerminalKind, TerminalKindExtensions, TextIndex,
+};
 use crate::parser::lexer::Lexer;
 use crate::parser::parser_support::context::ParserContext;
 use crate::parser::parser_support::parser_result::{
@@ -12,7 +14,7 @@ pub trait ParserFunction<P>
 where
     Self: Fn(&P, &mut ParserContext<'_>) -> ParserResult,
 {
-    fn parse(&self, parser: &P, input: &str) -> ParseOutput;
+    fn parse(&self, parser: &P, input: &str, topmost_kind: NonterminalKind) -> ParseOutput;
 }
 
 impl<P, F> ParserFunction<P> for F
@@ -21,7 +23,7 @@ where
     F: Fn(&P, &mut ParserContext<'_>) -> ParserResult,
 {
     #[allow(clippy::too_many_lines)]
-    fn parse(&self, parser: &P, input: &str) -> ParseOutput {
+    fn parse(&self, parser: &P, input: &str, topmost_kind: NonterminalKind) -> ParseOutput {
         let mut stream = ParserContext::new(input);
         let mut result = self(parser, &mut stream);
 
@@ -46,7 +48,7 @@ where
                 let mut new_children = nonterminal.children.clone();
                 new_children.extend(eof_trivia);
 
-                topmost.node = Node::nonterminal(nonterminal.kind, new_children);
+                topmost.node = Node::nonterminal(topmost_kind, new_children);
             }
         }
 
@@ -62,7 +64,7 @@ where
                 // trivia is already parsed twice, one for each branch. And there's a good reason: each branch might
                 // accept different trivia, so it's not clear what the combination of the two rules should return in a
                 // NoMatch. Therefore, we just parse it again. Note that trivia is anyway cached by the parser (#1119).
-                let mut trivia_nodes = if let ParserResult::Match(matched) =
+                let mut children = if let ParserResult::Match(matched) =
                     Lexer::leading_trivia(parser, &mut stream)
                 {
                     matched.nodes
@@ -71,7 +73,7 @@ where
                 };
 
                 let mut start = TextIndex::ZERO;
-                for edge in &trivia_nodes {
+                for edge in &children {
                     if let Node::Terminal(terminal) = &edge.node {
                         if terminal.kind.is_valid() {
                             start.advance_str(terminal.text.as_str());
@@ -85,14 +87,9 @@ where
                     TerminalKind::UNRECOGNIZED
                 };
                 let node = Node::terminal(kind, input.to_string());
-                let tree = if no_match.kind.is_none() || start.utf8 == 0 {
-                    node
-                } else {
-                    trivia_nodes.push(Edge::anonymous(node));
-                    Node::nonterminal(no_match.kind.unwrap(), trivia_nodes)
-                };
+                children.push(Edge::anonymous(node));
                 ParseOutput {
-                    tree,
+                    tree: Rc::new(NonterminalNode::new(topmost_kind, children)),
                     errors: vec![ParseError::new(
                         start..start + input.into(),
                         no_match.expected_terminals,
@@ -156,17 +153,17 @@ where
                     ));
 
                     ParseOutput {
-                        tree: Node::nonterminal(topmost_node.kind, new_children),
+                        tree: Rc::new(NonterminalNode::new(topmost_node.kind, new_children)),
                         errors,
                     }
                 } else {
-                    let tree = Node::Nonterminal(topmost_node);
+                    let tree = topmost_node;
                     let errors = stream.into_errors();
 
                     // Sanity check: Make sure that succesful parse is equivalent to not having any invalid nodes
                     debug_assert_eq!(
                         errors.is_empty(),
-                        tree.clone()
+                        Rc::clone(&tree)
                             .cursor_with_offset(TextIndex::ZERO)
                             .remaining_nodes()
                             .all(|edge| edge

crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/parser_result.rs

@@ -16,7 +16,6 @@ pub enum ParserResult {
 impl Default for ParserResult {
     fn default() -> Self {
         Self::NoMatch(NoMatch {
-            kind: None,
             expected_terminals: vec![],
         })
     }
@@ -35,13 +34,13 @@ impl ParserResult {
         ParserResult::IncompleteMatch(IncompleteMatch::new(nodes, expected_terminals))
     }
 
-    /// Whenever a parser didn't run because it's disabled due to versioning. Shorthand for `no_match(None, vec![])`.
+    /// Whenever a parser didn't run because it's disabled due to versioning. Shorthand for `no_match(vec![])`.
     pub fn disabled() -> Self {
-        Self::no_match(None, vec![])
+        Self::no_match(vec![])
     }
 
-    pub fn no_match(kind: Option<NonterminalKind>, expected_terminals: Vec<TerminalKind>) -> Self {
-        ParserResult::NoMatch(NoMatch::new(kind, expected_terminals))
+    pub fn no_match(expected_terminals: Vec<TerminalKind>) -> Self {
+        ParserResult::NoMatch(NoMatch::new(expected_terminals))
     }
 
     #[must_use]
@@ -62,9 +61,7 @@ impl ParserResult {
                 nodes: vec![Edge::anonymous(Node::nonterminal(new_kind, skipped.nodes))],
                 ..skipped
             }),
-            ParserResult::NoMatch(no_match) => {
-                ParserResult::no_match(Some(new_kind), no_match.expected_terminals)
-            }
+            ParserResult::NoMatch(no_match) => ParserResult::no_match(no_match.expected_terminals),
             ParserResult::PrattOperatorMatch(_) => {
                 unreachable!("PrattOperatorMatch cannot be converted to a nonterminal")
             }
@@ -243,18 +240,13 @@ impl IncompleteMatch {
 
 #[derive(PartialEq, Eq, Clone, Debug)]
 pub struct NoMatch {
-    /// The nonterminal kind this match is coming from
-    pub kind: Option<NonterminalKind>,
     /// Terminals that would have allowed for more progress. Collected for the purposes of error reporting.
     pub expected_terminals: Vec<TerminalKind>,
 }
 
 impl NoMatch {
-    pub fn new(kind: Option<NonterminalKind>, expected_terminals: Vec<TerminalKind>) -> Self {
-        Self {
-            kind,
-            expected_terminals,
-        }
+    pub fn new(expected_terminals: Vec<TerminalKind>) -> Self {
+        Self { expected_terminals }
     }
 }
 

crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/recovery.rs

@@ -105,7 +105,7 @@ impl ParserResult {
                 ParseResultKind::Incomplete => {
                     ParserResult::incomplete_match(nodes, expected_terminals)
                 }
-                ParseResultKind::NoMatch => ParserResult::no_match(None, expected_terminals),
+                ParseResultKind::NoMatch => ParserResult::no_match(expected_terminals),
             }
         }
     }

crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/repetition_helper.rs

@@ -26,10 +26,7 @@ impl<const MIN_COUNT: usize> RepetitionHelper<MIN_COUNT> {
 
             // Couldn't get a full match but we allow 0 items - return an empty match
             // so the parse is considered valid but note the expected terminals
-            ParserResult::NoMatch(NoMatch {
-                kind: _,
-                expected_terminals,
-            }) if MIN_COUNT == 0 => {
+            ParserResult::NoMatch(NoMatch { expected_terminals }) if MIN_COUNT == 0 => {
                 return ParserResult::r#match(vec![], expected_terminals);
             }
             // Don't try repeating if we don't have a full match and we require at least one
@@ -63,9 +60,7 @@ impl<const MIN_COUNT: usize> RepetitionHelper<MIN_COUNT> {
                     ParserResult::IncompleteMatch(IncompleteMatch {
                         expected_terminals, ..
                     })
-                    | ParserResult::NoMatch(NoMatch {
-                        expected_terminals, ..
-                    }),
+                    | ParserResult::NoMatch(NoMatch { expected_terminals }),
                 ) => {
                     input.rewind(save);
                     running.expected_terminals = expected_terminals;

crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/separated_helper.rs

@@ -89,7 +89,7 @@ impl SeparatedHelper {
                 }
                 ParserResult::NoMatch(no_match) => {
                     return if accum.is_empty() {
-                        ParserResult::no_match(None, no_match.expected_terminals)
+                        ParserResult::no_match(no_match.expected_terminals)
                     } else {
                         ParserResult::incomplete_match(accum, no_match.expected_terminals)
                     };

crates/codegen/runtime/cargo/wasm/src/runtime/interface/compilation.wit.jinja2

@@ -1,6 +1,6 @@
 interface compilation {
     use bindings.{binding-graph};
-    use cst.{node, cursor};
+    use cst.{nonterminal-node, cursor};
 
     /// A builder for creating compilation units.
     /// Allows incrementally building a transitive list of all files and their imports.
@@ -60,7 +60,7 @@ interface compilation {
         id: func() -> string;
 
         /// Returns the syntax tree of this file.
-        tree: func() -> node;
+        tree: func() -> nonterminal-node;
 
         /// Creates a cursor for traversing the syntax tree of this file.
         create-tree-cursor: func() -> cursor;

crates/codegen/runtime/cargo/wasm/src/runtime/interface/parser.wit.jinja2

@@ -1,5 +1,5 @@
 interface parser {
-    use cst.{cursor, node, nonterminal-kind, text-range};
+    use cst.{cursor, nonterminal-node, nonterminal-kind, text-range};
 
     /// A parser instance that can parse source code into syntax trees.
     /// Each parser is configured for a specific language version and grammar.
@@ -32,7 +32,7 @@ interface parser {
     resource parse-output {
         /// Returns the root node of the parsed syntax tree.
         /// Even if there are parsing errors, a partial tree will still be available.
-        tree: func() -> node;
+        tree: func() -> nonterminal-node;
 
         /// Returns a list of all parsing errors encountered.
         /// An empty list indicates successful parsing with no errors.

crates/codegen/runtime/cargo/wasm/src/runtime/wrappers/compilation/mod.rs

@@ -11,7 +11,9 @@ mod ffi {
         Guest, GuestCompilationUnit, GuestFile, GuestInternalCompilationBuilder,
         InternalCompilationBuilder, InternalCompilationBuilderBorrow,
     };
-    pub use crate::wasm_crate::bindgen::exports::nomic_foundation::slang::cst::{Cursor, Node};
+    pub use crate::wasm_crate::bindgen::exports::nomic_foundation::slang::cst::{
+        Cursor, NonterminalNode,
+    };
 }
 
 mod rust {
@@ -110,7 +112,7 @@ define_rc_wrapper! { File {
         self._borrow_ffi().id().to_owned()
     }
 
-    fn tree(&self) -> ffi::Node {
+    fn tree(&self) -> ffi::NonterminalNode {
         self._borrow_ffi().tree().to_owned()._into_ffi()
     }