fix: newlines not being converted into whitespace properly in paragraphs

nvim-neorg · Aug 12, 2024 · 6137299 · 6137299
1 parent 8c18777
commit 6137299
Show file tree

Hide file tree

Showing 7 changed files with 90 additions and 166 deletions.
diff --git a/proptest-regressions/lib.txt b/proptest-regressions/lib.txt
@@ -6,3 +6,4 @@
 # everyone who runs the test benefits from these saved cases.
 cc fb9b5df4fe46fe331cc3aa40bba6501c1c603084688fd02dda6d1c73106c1324 # shrinks to tag_name = "A", parameter = "A", multi_parameter = "\u{b}"
 cc 28afae9872324ba0632a8023219e32939580363ce8b99752dc19fae0ac5b63d1 # shrinks to paragraph_content = " "
+cc 5b31c9987c98fc0e4faa50b782e5952e0948d2c2a60dd29081c8f54c75b4b52c # shrinks to tag_name = "ø", parameter = "a", multi_parameter = " "
diff --git a/src/lib.rs b/src/lib.rs
@@ -532,7 +532,6 @@ mod tests {
             "*hello*, world!",
             "*/hello/*, world!",
             "*hi!* how are you?",
-
             "this *is a test",
             "this *is/ a test",
             "this *is*/ a test",

diff --git a/src/snapshots/rust_norg__tests__modifiers.snap b/src/snapshots/rust_norg__tests__modifiers.snap
@@ -89,3 +89,70 @@ expression: examples
       - Token: Whitespace
       - Token:
           Text: you?
+- - Paragraph:
+      - Token:
+          Text: this
+      - Token: Whitespace
+      - Token:
+          Special: "*"
+      - Token:
+          Text: is
+      - Token: Whitespace
+      - Token:
+          Text: a
+      - Token: Whitespace
+      - Token:
+          Text: test
+- - Paragraph:
+      - Token:
+          Text: this
+      - Token: Whitespace
+      - Token:
+          Special: "*"
+      - Token:
+          Text: is
+      - Token:
+          Special: /
+      - Token: Whitespace
+      - Token:
+          Text: a
+      - Token: Whitespace
+      - Token:
+          Text: test
+- - Paragraph:
+      - Token:
+          Text: this
+      - Token: Whitespace
+      - AttachedModifier:
+          modifier_type: "*"
+          content:
+            - Token:
+                Text: is
+      - Token:
+          Special: /
+      - Token: Whitespace
+      - Token:
+          Text: a
+      - Token: Whitespace
+      - Token:
+          Text: test
+- - Paragraph:
+      - Token:
+          Text: this
+      - Token: Whitespace
+      - AttachedModifier:
+          modifier_type: "*"
+          content:
+            - AttachedModifier:
+                modifier_type: /
+                content:
+                  - Token:
+                      Text: is
+      - Token:
+          Special: /
+      - Token: Whitespace
+      - Token:
+          Text: a
+      - Token: Whitespace
+      - Token:
+          Text: test
diff --git a/src/snapshots/rust_norg__tests__modifiers.snap.new b/src/snapshots/rust_norg__tests__modifiers.snap.new
diff --git a/src/snapshots/rust_norg__tests__paragraphs.snap b/src/snapshots/rust_norg__tests__paragraphs.snap
@@ -33,6 +33,7 @@ expression: examples
                 Text: hello
             - Token:
                 Special: ","
+            - Token: Whitespace
             - Token:
                 Text: world
             - Token:
@@ -46,6 +47,7 @@ expression: examples
 - - Paragraph:
       - Token:
           Text: paragraph
+      - Token: Whitespace
       - Token:
           Text: here
   - Paragraph:
@@ -54,6 +56,7 @@ expression: examples
       - Token: Whitespace
       - Token:
           Text: paragraph
+      - Token: Whitespace
       - Token:
           Text: here
       - Token:

diff --git a/src/stage_2.rs b/src/stage_2.rs
@@ -43,7 +43,9 @@ fn tokens_to_paragraph_segment(tokens: Vec<NorgToken>) -> ParagraphTokenList {
         .into_iter()
         .peekable()
         .batching(|it| match it.next() {
-            Some(NorgToken::Whitespace(_)) => Some(ParagraphSegmentToken::Whitespace),
+            Some(NorgToken::SingleNewline) | Some(NorgToken::Whitespace(_)) => {
+                Some(ParagraphSegmentToken::Whitespace)
+            }
             Some(NorgToken::Special(c)) => Some(ParagraphSegmentToken::Special(c)),
             Some(NorgToken::Escape(c)) => Some(ParagraphSegmentToken::Escape(c)),
             Some(NorgToken::Regular(c)) => {
@@ -401,9 +403,9 @@ pub fn stage_2() -> impl Parser<NorgToken, Vec<NorgBlock>, Error = chumsky::erro
                 NorgToken::Newlines(_) => {
                     NorgBlock::ParagraphSegmentEnd(tokens_to_paragraph_segment(content))
                 }
-                NorgToken::SingleNewline => {
-                    NorgBlock::ParagraphSegment(tokens_to_paragraph_segment(content))
-                }
+                NorgToken::SingleNewline => NorgBlock::ParagraphSegment(
+                    tokens_to_paragraph_segment(content.into_iter().chain(trailing).collect()),
+                ),
                 _ => unreachable!(),
             })
             .labelled("paragraph_segment"),

diff --git a/src/stage_3.rs b/src/stage_3.rs
@@ -243,7 +243,10 @@ fn paragraph_rollup_candidates(
                     content,
                 })
             } else {
-                Err(Simple::custom(span, "differing opening and closing modifiers found"))
+                Err(Simple::custom(
+                    span,
+                    "differing opening and closing modifiers found",
+                ))
             }
         })
     });
@@ -453,7 +456,15 @@ pub fn stage_3(
                 .chain(paragraph_segment_end.or_not()),
             paragraph_segment_end,
         ))
-            .map(|tokens| NorgASTFlat::Paragraph(parse_paragraph(tokens).unwrap()));
+            .map(|mut tokens| {
+                // Trim trailing whitespace (both user-induced but also induced by us when
+                // converting single newlines to whitespace).
+                if let Some(ParagraphSegmentToken::Whitespace) = tokens.last() {
+                    tokens.pop();
+                }
+
+                NorgASTFlat::Paragraph(parse_paragraph(tokens).unwrap())
+            });
 
         let nestable_detached_modifier = select! {
             NorgBlock::NestableDetachedModifier { modifier_type: '-', level, extension_section } => (NestableDetachedModifier::UnorderedList, level, extension_section),