diff --git a/proptest-regressions/lib.txt b/proptest-regressions/lib.txt index 5697447..557f8ed 100644 --- a/proptest-regressions/lib.txt +++ b/proptest-regressions/lib.txt @@ -6,3 +6,4 @@ # everyone who runs the test benefits from these saved cases. cc fb9b5df4fe46fe331cc3aa40bba6501c1c603084688fd02dda6d1c73106c1324 # shrinks to tag_name = "A", parameter = "A", multi_parameter = "\u{b}" cc 28afae9872324ba0632a8023219e32939580363ce8b99752dc19fae0ac5b63d1 # shrinks to paragraph_content = " " +cc 5b31c9987c98fc0e4faa50b782e5952e0948d2c2a60dd29081c8f54c75b4b52c # shrinks to tag_name = "ΓΈ", parameter = "a", multi_parameter = " " diff --git a/src/lib.rs b/src/lib.rs index c666476..a499574 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -532,7 +532,6 @@ mod tests { "*hello*, world!", "*/hello/*, world!", "*hi!* how are you?", - "this *is a test", "this *is/ a test", "this *is*/ a test", diff --git a/src/snapshots/rust_norg__tests__modifiers.snap b/src/snapshots/rust_norg__tests__modifiers.snap index 17d7599..5535451 100644 --- a/src/snapshots/rust_norg__tests__modifiers.snap +++ b/src/snapshots/rust_norg__tests__modifiers.snap @@ -89,3 +89,70 @@ expression: examples - Token: Whitespace - Token: Text: you? +- - Paragraph: + - Token: + Text: this + - Token: Whitespace + - Token: + Special: "*" + - Token: + Text: is + - Token: Whitespace + - Token: + Text: a + - Token: Whitespace + - Token: + Text: test +- - Paragraph: + - Token: + Text: this + - Token: Whitespace + - Token: + Special: "*" + - Token: + Text: is + - Token: + Special: / + - Token: Whitespace + - Token: + Text: a + - Token: Whitespace + - Token: + Text: test +- - Paragraph: + - Token: + Text: this + - Token: Whitespace + - AttachedModifier: + modifier_type: "*" + content: + - Token: + Text: is + - Token: + Special: / + - Token: Whitespace + - Token: + Text: a + - Token: Whitespace + - Token: + Text: test +- - Paragraph: + - Token: + Text: this + - Token: Whitespace + - AttachedModifier: + modifier_type: "*" + content: + - AttachedModifier: + modifier_type: / + content: + - Token: + Text: is + - Token: + Special: / + - Token: Whitespace + - Token: + Text: a + - Token: Whitespace + - Token: + Text: test diff --git a/src/snapshots/rust_norg__tests__modifiers.snap.new b/src/snapshots/rust_norg__tests__modifiers.snap.new deleted file mode 100644 index d8084bd..0000000 --- a/src/snapshots/rust_norg__tests__modifiers.snap.new +++ /dev/null @@ -1,159 +0,0 @@ ---- -source: src/lib.rs -assertion_line: 547 -expression: examples ---- -- - Paragraph: - - Token: - Text: this - - Token: Whitespace - - AttachedModifier: - modifier_type: "*" - content: - - Token: - Text: is - - Token: Whitespace - - Token: - Text: a - - Token: Whitespace - - Token: - Text: test -- - Paragraph: - - Token: - Text: hello - - Token: - Special: "," - - Token: Whitespace - - AttachedModifier: - modifier_type: "*" - content: - - Token: - Text: world - - Token: - Special: "!" -- - Paragraph: - - AttachedModifier: - modifier_type: "*" - content: - - Token: - Text: hello - - Token: - Special: "," - - Token: Whitespace - - Token: - Text: world - - Token: - Special: "!" -- - Paragraph: - - AttachedModifier: - modifier_type: "*" - content: - - Token: - Text: hello - - Token: - Special: "," - - Token: Whitespace - - Token: - Text: world - - Token: - Special: "!" -- - Paragraph: - - AttachedModifier: - modifier_type: "*" - content: - - AttachedModifier: - modifier_type: / - content: - - Token: - Text: hello - - Token: - Special: "," - - Token: Whitespace - - Token: - Text: world - - Token: - Special: "!" -- - Paragraph: - - AttachedModifier: - modifier_type: "*" - content: - - Token: - Text: hi - - Token: - Special: "!" - - Token: Whitespace - - Token: - Text: how - - Token: Whitespace - - Token: - Text: are - - Token: Whitespace - - Token: - Text: you? -- - Paragraph: - - Token: - Text: this - - Token: Whitespace - - Token: - Special: "*" - - Token: - Text: is - - Token: Whitespace - - Token: - Text: a - - Token: Whitespace - - Token: - Text: test -- - Paragraph: - - Token: - Text: this - - Token: Whitespace - - Token: - Special: "*" - - Token: - Text: is - - Token: - Special: / - - Token: Whitespace - - Token: - Text: a - - Token: Whitespace - - Token: - Text: test -- - Paragraph: - - Token: - Text: this - - Token: Whitespace - - AttachedModifier: - modifier_type: "*" - content: - - Token: - Text: is - - Token: - Special: / - - Token: Whitespace - - Token: - Text: a - - Token: Whitespace - - Token: - Text: test -- - Paragraph: - - Token: - Text: this - - Token: Whitespace - - AttachedModifier: - modifier_type: "*" - content: - - AttachedModifier: - modifier_type: / - content: - - Token: - Text: is - - Token: - Special: / - - Token: Whitespace - - Token: - Text: a - - Token: Whitespace - - Token: - Text: test diff --git a/src/snapshots/rust_norg__tests__paragraphs.snap b/src/snapshots/rust_norg__tests__paragraphs.snap index a18cd11..92f778e 100644 --- a/src/snapshots/rust_norg__tests__paragraphs.snap +++ b/src/snapshots/rust_norg__tests__paragraphs.snap @@ -33,6 +33,7 @@ expression: examples Text: hello - Token: Special: "," + - Token: Whitespace - Token: Text: world - Token: @@ -46,6 +47,7 @@ expression: examples - - Paragraph: - Token: Text: paragraph + - Token: Whitespace - Token: Text: here - Paragraph: @@ -54,6 +56,7 @@ expression: examples - Token: Whitespace - Token: Text: paragraph + - Token: Whitespace - Token: Text: here - Token: diff --git a/src/stage_2.rs b/src/stage_2.rs index bd2232c..4bf6527 100644 --- a/src/stage_2.rs +++ b/src/stage_2.rs @@ -43,7 +43,9 @@ fn tokens_to_paragraph_segment(tokens: Vec) -> ParagraphTokenList { .into_iter() .peekable() .batching(|it| match it.next() { - Some(NorgToken::Whitespace(_)) => Some(ParagraphSegmentToken::Whitespace), + Some(NorgToken::SingleNewline) | Some(NorgToken::Whitespace(_)) => { + Some(ParagraphSegmentToken::Whitespace) + } Some(NorgToken::Special(c)) => Some(ParagraphSegmentToken::Special(c)), Some(NorgToken::Escape(c)) => Some(ParagraphSegmentToken::Escape(c)), Some(NorgToken::Regular(c)) => { @@ -401,9 +403,9 @@ pub fn stage_2() -> impl Parser, Error = chumsky::erro NorgToken::Newlines(_) => { NorgBlock::ParagraphSegmentEnd(tokens_to_paragraph_segment(content)) } - NorgToken::SingleNewline => { - NorgBlock::ParagraphSegment(tokens_to_paragraph_segment(content)) - } + NorgToken::SingleNewline => NorgBlock::ParagraphSegment( + tokens_to_paragraph_segment(content.into_iter().chain(trailing).collect()), + ), _ => unreachable!(), }) .labelled("paragraph_segment"), diff --git a/src/stage_3.rs b/src/stage_3.rs index fbbe1ee..5bb4e3f 100644 --- a/src/stage_3.rs +++ b/src/stage_3.rs @@ -243,7 +243,10 @@ fn paragraph_rollup_candidates( content, }) } else { - Err(Simple::custom(span, "differing opening and closing modifiers found")) + Err(Simple::custom( + span, + "differing opening and closing modifiers found", + )) } }) }); @@ -453,7 +456,15 @@ pub fn stage_3( .chain(paragraph_segment_end.or_not()), paragraph_segment_end, )) - .map(|tokens| NorgASTFlat::Paragraph(parse_paragraph(tokens).unwrap())); + .map(|mut tokens| { + // Trim trailing whitespace (both user-induced but also induced by us when + // converting single newlines to whitespace). + if let Some(ParagraphSegmentToken::Whitespace) = tokens.last() { + tokens.pop(); + } + + NorgASTFlat::Paragraph(parse_paragraph(tokens).unwrap()) + }); let nestable_detached_modifier = select! { NorgBlock::NestableDetachedModifier { modifier_type: '-', level, extension_section } => (NestableDetachedModifier::UnorderedList, level, extension_section),