Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Change ParseOutput to return a NonTerminal instead of a Node #1187

Merged
merged 25 commits into from
Jan 13, 2025
Merged
Show file tree
Hide file tree
Changes from 21 commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
5db05e8
First attempt: re-parse leading trivia and advance index accordingly
beta-ziliani Nov 21, 2024
86540f4
Adding nodes in NoMatch
beta-ziliani Nov 25, 2024
a74ea7d
NoMatch with tracking information about trivia and originator non-ter…
beta-ziliani Nov 27, 2024
cf6b964
Missing files
beta-ziliani Nov 27, 2024
3fbefa4
Revert to not carry the trivia in the NoMatch result
beta-ziliani Nov 28, 2024
24aff8c
Revert little change with no semantic difference
beta-ziliani Nov 28, 2024
ecd5062
Fixing comment
beta-ziliani Nov 28, 2024
4113609
revert of unnecessary change
beta-ziliani Nov 28, 2024
7495413
Adding comment
beta-ziliani Dec 10, 2024
4dc2d0b
Adding changeset
beta-ziliani Dec 10, 2024
8983d3c
Merge remote-tracking branch 'origin/main' into beta/841
beta-ziliani Dec 10, 2024
72d4858
fixup
beta-ziliani Dec 11, 2024
26d5e00
Changing parse_result to return a NonTerminalNode instead of a Node
beta-ziliani Dec 11, 2024
160e02e
Generated examples
beta-ziliani Dec 11, 2024
95137a0
Missing files
beta-ziliani Dec 12, 2024
e8c1398
solved merge conflict
beta-ziliani Dec 13, 2024
05d405a
Moving clone to clients
beta-ziliani Dec 19, 2024
9ee0996
Pulling the kind away of nomatch to the parse() function
beta-ziliani Jan 8, 2025
299d966
Changes in generated files
beta-ziliani Jan 8, 2025
3f174c8
Fixing name
beta-ziliani Jan 8, 2025
6a5c889
merging with main
beta-ziliani Jan 8, 2025
1fb766e
Applying suggestions from code review
beta-ziliani Jan 10, 2025
850b5f9
Changing the type of File.tree
beta-ziliani Jan 10, 2025
90d72b7
Adding missing files
beta-ziliani Jan 10, 2025
6615c18
Create young-bottles-beam.md
beta-ziliani Jan 13, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
use std::collections::BTreeMap;
use std::rc::Rc;

use metaslang_cst::nodes::Node;
use semver::Version;

use crate::compilation::{CompilationUnit, File};
Expand Down Expand Up @@ -43,7 +44,10 @@ impl InternalCompilationBuilder {

let import_paths = self.imports.extract(parse_output.create_tree_cursor());

let file = File::new(id.clone(), parse_output.tree().clone());
let file = File::new(
OmarTawfik marked this conversation as resolved.
Show resolved Hide resolved
id.clone(),
Node::Nonterminal(Rc::clone(parse_output.tree())),
);
self.files.insert(id, file);

AddFileResponse { import_paths }
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ pub(crate) trait Lexer {
.is_some_and(|t| t.accepted_as(kind))
{
input.set_position(start);
return ParserResult::no_match(None, vec![kind]);
return ParserResult::no_match(vec![kind]);
}
let end = input.position();

Expand Down Expand Up @@ -129,7 +129,7 @@ pub(crate) trait Lexer {
.is_some_and(|t| t.accepted_as(kind))
{
input.set_position(restore);
return ParserResult::no_match(None, vec![kind]);
return ParserResult::no_match(vec![kind]);
}
let end = input.position();
children.push(Edge::anonymous(Node::terminal(
Expand Down
Original file line number Diff line number Diff line change
@@ -1,14 +1,16 @@
use crate::cst::{Cursor, Node, TextIndex};
use std::rc::Rc;

use crate::cst::{Cursor, NonterminalNode, TextIndex};
use crate::parser::ParseError;

#[derive(Clone, Debug, PartialEq)]
pub struct ParseOutput {
pub(crate) tree: Node,
pub(crate) tree: Rc<NonterminalNode>,
pub(crate) errors: Vec<ParseError>,
}

impl ParseOutput {
pub fn tree(&self) -> &Node {
pub fn tree(&self) -> &Rc<NonterminalNode> {
&self.tree
}

Expand All @@ -22,6 +24,6 @@ impl ParseOutput {

/// Creates a cursor that starts at the root of the parse tree.
pub fn create_tree_cursor(&self) -> Cursor {
self.tree.clone().cursor_with_offset(TextIndex::ZERO)
Rc::clone(&self.tree).cursor_with_offset(TextIndex::ZERO)
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,7 @@ impl Parser {
{%- else -%}
match kind {
{%- for parser_name, _ in model.parser.parser_functions -%}
NonterminalKind::{{ parser_name }} => Self::{{ parser_name | snake_case }}.parse(self, input),
NonterminalKind::{{ parser_name }} => Self::{{ parser_name | snake_case }}.parse(self, input, kind),
{%- endfor -%}
}
{%- endif -%}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
use std::rc::Rc;

use crate::cst::{Edge, Node, TerminalKind, TerminalKindExtensions, TextIndex};
use crate::cst::{
Edge, Node, NonterminalKind, NonterminalNode, TerminalKind, TerminalKindExtensions, TextIndex,
};
use crate::parser::lexer::Lexer;
use crate::parser::parser_support::context::ParserContext;
use crate::parser::parser_support::parser_result::{
Expand All @@ -12,7 +14,7 @@ pub trait ParserFunction<P>
where
Self: Fn(&P, &mut ParserContext<'_>) -> ParserResult,
{
fn parse(&self, parser: &P, input: &str) -> ParseOutput;
fn parse(&self, parser: &P, input: &str, expected: NonterminalKind) -> ParseOutput;
OmarTawfik marked this conversation as resolved.
Show resolved Hide resolved
}

impl<P, F> ParserFunction<P> for F
Expand All @@ -21,7 +23,7 @@ where
F: Fn(&P, &mut ParserContext<'_>) -> ParserResult,
{
#[allow(clippy::too_many_lines)]
fn parse(&self, parser: &P, input: &str) -> ParseOutput {
fn parse(&self, parser: &P, input: &str, expected: NonterminalKind) -> ParseOutput {
let mut stream = ParserContext::new(input);
let mut result = self(parser, &mut stream);

Expand All @@ -46,7 +48,7 @@ where
let mut new_children = nonterminal.children.clone();
new_children.extend(eof_trivia);

topmost.node = Node::nonterminal(nonterminal.kind, new_children);
topmost.node = Node::nonterminal(expected, new_children);
}
}

Expand Down Expand Up @@ -85,14 +87,9 @@ where
TerminalKind::UNRECOGNIZED
};
let node = Node::terminal(kind, input.to_string());
let tree = if no_match.kind.is_none() || start.utf8 == 0 {
node
} else {
trivia_nodes.push(Edge::anonymous(node));
Node::nonterminal(no_match.kind.unwrap(), trivia_nodes)
};
trivia_nodes.push(Edge::anonymous(node));
ParseOutput {
tree,
tree: Rc::new(NonterminalNode::new(expected, trivia_nodes)),
OmarTawfik marked this conversation as resolved.
Show resolved Hide resolved
errors: vec![ParseError::new(
start..start + input.into(),
no_match.expected_terminals,
Expand Down Expand Up @@ -156,17 +153,17 @@ where
));

ParseOutput {
tree: Node::nonterminal(topmost_node.kind, new_children),
tree: Rc::new(NonterminalNode::new(topmost_node.kind, new_children)),
errors,
}
} else {
let tree = Node::Nonterminal(topmost_node);
let tree = topmost_node;
let errors = stream.into_errors();

// Sanity check: Make sure that succesful parse is equivalent to not having any invalid nodes
debug_assert_eq!(
errors.is_empty(),
tree.clone()
Rc::clone(&tree)
.cursor_with_offset(TextIndex::ZERO)
.remaining_nodes()
.all(|edge| edge
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@ pub enum ParserResult {
impl Default for ParserResult {
fn default() -> Self {
Self::NoMatch(NoMatch {
kind: None,
expected_terminals: vec![],
})
}
Expand All @@ -35,13 +34,13 @@ impl ParserResult {
ParserResult::IncompleteMatch(IncompleteMatch::new(nodes, expected_terminals))
}

/// Whenever a parser didn't run because it's disabled due to versioning. Shorthand for `no_match(None, vec![])`.
/// Whenever a parser didn't run because it's disabled due to versioning. Shorthand for `no_match(vec![])`.
pub fn disabled() -> Self {
Self::no_match(None, vec![])
Self::no_match(vec![])
}

pub fn no_match(kind: Option<NonterminalKind>, expected_terminals: Vec<TerminalKind>) -> Self {
ParserResult::NoMatch(NoMatch::new(kind, expected_terminals))
pub fn no_match(expected_terminals: Vec<TerminalKind>) -> Self {
ParserResult::NoMatch(NoMatch::new(expected_terminals))
}

#[must_use]
Expand All @@ -62,9 +61,7 @@ impl ParserResult {
nodes: vec![Edge::anonymous(Node::nonterminal(new_kind, skipped.nodes))],
..skipped
}),
ParserResult::NoMatch(no_match) => {
ParserResult::no_match(Some(new_kind), no_match.expected_terminals)
}
ParserResult::NoMatch(no_match) => ParserResult::no_match(no_match.expected_terminals),
ParserResult::PrattOperatorMatch(_) => {
unreachable!("PrattOperatorMatch cannot be converted to a nonterminal")
}
Expand Down Expand Up @@ -243,18 +240,13 @@ impl IncompleteMatch {

#[derive(PartialEq, Eq, Clone, Debug)]
pub struct NoMatch {
/// The nonterminal kind this match is coming from
pub kind: Option<NonterminalKind>,
/// Terminals that would have allowed for more progress. Collected for the purposes of error reporting.
pub expected_terminals: Vec<TerminalKind>,
}

impl NoMatch {
pub fn new(kind: Option<NonterminalKind>, expected_terminals: Vec<TerminalKind>) -> Self {
Self {
kind,
expected_terminals,
}
pub fn new(expected_terminals: Vec<TerminalKind>) -> Self {
Self { expected_terminals }
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@ impl ParserResult {
ParseResultKind::Incomplete => {
ParserResult::incomplete_match(nodes, expected_terminals)
}
ParseResultKind::NoMatch => ParserResult::no_match(None, expected_terminals),
ParseResultKind::NoMatch => ParserResult::no_match(expected_terminals),
}
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,10 +26,7 @@ impl<const MIN_COUNT: usize> RepetitionHelper<MIN_COUNT> {

// Couldn't get a full match but we allow 0 items - return an empty match
// so the parse is considered valid but note the expected terminals
ParserResult::NoMatch(NoMatch {
kind: _,
expected_terminals,
}) if MIN_COUNT == 0 => {
ParserResult::NoMatch(NoMatch { expected_terminals }) if MIN_COUNT == 0 => {
return ParserResult::r#match(vec![], expected_terminals);
}
// Don't try repeating if we don't have a full match and we require at least one
Expand Down Expand Up @@ -63,9 +60,7 @@ impl<const MIN_COUNT: usize> RepetitionHelper<MIN_COUNT> {
ParserResult::IncompleteMatch(IncompleteMatch {
expected_terminals, ..
})
| ParserResult::NoMatch(NoMatch {
expected_terminals, ..
}),
| ParserResult::NoMatch(NoMatch { expected_terminals }),
) => {
input.rewind(save);
running.expected_terminals = expected_terminals;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ impl SeparatedHelper {
}
ParserResult::NoMatch(no_match) => {
return if accum.is_empty() {
ParserResult::no_match(None, no_match.expected_terminals)
ParserResult::no_match(no_match.expected_terminals)
} else {
ParserResult::incomplete_match(accum, no_match.expected_terminals)
};
Expand Down

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
interface parser {
use cst.{cursor, node, nonterminal-kind, text-range};
use cst.{cursor, nonterminal-node, nonterminal-kind, text-range};

/// A parser instance that can parse source code into syntax trees.
/// Each parser is configured for a specific language version and grammar.
Expand Down Expand Up @@ -32,7 +32,7 @@ interface parser {
resource parse-output {
/// Returns the root node of the parsed syntax tree.
/// Even if there are parsing errors, a partial tree will still be available.
tree: func() -> node;
tree: func() -> nonterminal-node;

/// Returns a list of all parsing errors encountered.
/// An empty list indicates successful parsing with no errors.
Expand Down
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
use std::rc::Rc;

use crate::wasm_crate::utils::{define_wrapper, FromFFI, IntoFFI};

mod ffi {
pub use crate::wasm_crate::bindgen::exports::nomic_foundation::slang::cst::{
Cursor, Node, TextRange,
Cursor, NonterminalNode, TextRange,
};
pub use crate::wasm_crate::bindgen::exports::nomic_foundation::slang::parser::{
Guest, GuestParseError, GuestParseOutput, GuestParser, NonterminalKind, ParseError,
Expand Down Expand Up @@ -70,8 +72,8 @@ define_wrapper! { ParseError {
//================================================

define_wrapper! { ParseOutput {
fn tree(&self) -> ffi::Node {
self._borrow_ffi().tree().clone()._into_ffi()
fn tree(&self) -> ffi::NonterminalNode {
Rc::clone(self._borrow_ffi().tree())._into_ffi()
}

fn errors(&self) -> Vec<ffi::ParseError> {
Expand Down

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions crates/metaslang/cst/generated/public_api.txt

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

20 changes: 13 additions & 7 deletions crates/metaslang/cst/src/nodes.rs
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,18 @@ pub struct Edge<T: KindTypes> {
pub node: Node<T>,
}

impl<T: KindTypes> NonterminalNode<T> {
pub fn new(kind: T::NonterminalKind, children: Vec<Edge<T>>) -> Self {
let text_len = children.iter().map(|edge| edge.text_len()).sum();

NonterminalNode {
kind,
text_len,
children,
}
}
}

impl<T: KindTypes> Edge<T> {
/// Creates an anonymous node (without a label).
pub fn anonymous(node: Node<T>) -> Self {
Expand All @@ -81,13 +93,7 @@ impl<T: KindTypes> std::ops::Deref for Edge<T> {

impl<T: KindTypes> Node<T> {
pub fn nonterminal(kind: T::NonterminalKind, children: Vec<Edge<T>>) -> Self {
let text_len = children.iter().map(|edge| edge.text_len()).sum();

Self::Nonterminal(Rc::new(NonterminalNode {
kind,
text_len,
children,
}))
Self::Nonterminal(Rc::new(NonterminalNode::new(kind, children)))
}

pub fn terminal(kind: T::TerminalKind, text: String) -> Self {
Expand Down

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,8 @@ pub fn add_built_ins(
let parser = Parser::create(version)?;
let parse_output = parser.parse(Parser::ROOT_KIND, source);

let built_ins_cursor = transform(parse_output.tree()).cursor_with_offset(TextIndex::ZERO);
let built_ins_cursor = transform(&Node::Nonterminal(Rc::clone(parse_output.tree())))
.cursor_with_offset(TextIndex::ZERO);

binding_graph.add_system_file("built_ins.sol", built_ins_cursor);
Ok(())
Expand Down
Loading
Loading