diff --git a/Cargo.lock b/Cargo.lock index e664f2a..f5ee7fd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,6 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "ahash" @@ -212,9 +212,9 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "proc-macro2" -version = "1.0.82" +version = "1.0.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ad3d49ab951a01fbaafe34f2ec74122942fe18a3f9814c3268f1bb72042131b" +checksum = "60946a68e5f9d28b0dc1c21bb8a97ee7d018a8b322fa57838ba31cc878e22d99" dependencies = [ "unicode-ident", ] @@ -256,9 +256,9 @@ checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" [[package]] name = "quote" -version = "1.0.36" +version = "1.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" +checksum = "0e4dccaaaf89514f546c693ddc140f729f958c247918a13380cccc6078391acc" dependencies = [ "proc-macro2", ] @@ -317,6 +317,7 @@ dependencies = [ "itertools", "proptest", "serde", + "textwrap", "unicode_categories", ] @@ -371,6 +372,12 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fa42c91313f1d05da9b26f267f931cf178d4aba455b4c4622dd7355eb80c6640" +[[package]] +name = "smawk" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c388c1b5e93756d0c740965c41e8822f866621d41acbdf6336a6a168f8840c" + [[package]] name = "stacker" version = "0.1.15" @@ -386,9 +393,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.61" +version = "2.0.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c993ed8ccba56ae856363b1845da7266a7cb78e1d146c8a32d54b45a8b831fc9" +checksum = "36147f1a48ae0ec2b5b3bc5b537d267457555a10dc06f3dbc8cb11ba3006d3b1" dependencies = [ "proc-macro2", "quote", @@ -407,6 +414,17 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "textwrap" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23d434d3f8967a09480fb04132ebe0a3e088c173e6d0ee7897abbdf4eab0f8b9" +dependencies = [ + "smawk", + "unicode-linebreak", + "unicode-width", +] + [[package]] name = "unarray" version = "0.1.4" @@ -419,6 +437,18 @@ version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +[[package]] +name = "unicode-linebreak" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b09c83c3c29d37506a3e260c08c03743a6bb66a9cd432c6934ab501a190571f" + +[[package]] +name = "unicode-width" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" + [[package]] name = "unicode_categories" version = "0.1.1" diff --git a/Cargo.toml b/Cargo.toml index b0c334e..1afe57f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,6 +7,7 @@ edition = "2021" chumsky = "0.9.3" itertools = "0.13.0" serde = { version = "1.0.203", features = ["derive"] } +textwrap = "0.16.1" unicode_categories = "0.1.1" [dev-dependencies] diff --git a/src/lib.rs b/src/lib.rs index c6623a0..603c261 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2,7 +2,7 @@ use chumsky::Parser as _; use error::NorgParseError; pub use crate::stage_1::stage_1; -use crate::stage_2::stage_2; +pub use crate::stage_2::stage_2; use crate::stage_4::stage_4; pub use crate::stage_2::ParagraphSegmentToken; diff --git a/src/snapshots/rust_norg__tests__ranged_verbatim_tags.snap b/src/snapshots/rust_norg__tests__ranged_verbatim_tags.snap index 5531c7e..2361962 100644 --- a/src/snapshots/rust_norg__tests__ranged_verbatim_tags.snap +++ b/src/snapshots/rust_norg__tests__ranged_verbatim_tags.snap @@ -23,4 +23,4 @@ expression: examples - first-parameter - "#&*(&$!)" - third-parameter - content: "function hello()\nprint(\"Hello World\")\nend hello()\n" + content: "function hello()\n print(\"Hello World\")\nend\n\nhello()\n" diff --git a/src/stage_1.rs b/src/stage_1.rs index 53f19d2..11621cd 100644 --- a/src/stage_1.rs +++ b/src/stage_1.rs @@ -29,7 +29,7 @@ impl std::fmt::Display for NorgToken { Self::End(c) => write!(f, "{}end", c), Self::Eof => f.write_char('\0'), Self::Escape(c) => write!(f, "\\{}", c), - Self::Newlines(count) => f.write_str(&" ".repeat(*count as usize)), + Self::Newlines(count) => f.write_str(&"\n".repeat(*count as usize)), Self::Regular(c) | Self::Special(c) => f.write_char(*c), Self::SingleNewline => f.write_char('\n'), Self::Whitespace(count) => f.write_str(&" ".repeat(*count as usize)), @@ -61,13 +61,11 @@ pub fn stage_1() -> impl Parser, Error = chumsky::error::Si }); let newline = parse_newline - .then_ignore(ws.repeated()) .to(NorgToken::SingleNewline); let newlines = parse_newline .repeated() .at_least(2) - .then_ignore(ws.repeated()) .map(|content| NorgToken::Newlines(content.len() as u16)); let special = one_of(SPECIAL_CHARS).map(NorgToken::Special); diff --git a/src/stage_3.rs b/src/stage_3.rs index 158682e..6e56490 100644 --- a/src/stage_3.rs +++ b/src/stage_3.rs @@ -3,6 +3,7 @@ use std::fmt::Write; use chumsky::prelude::*; use itertools::Itertools; use serde::Serialize; +use textwrap::dedent; use crate::stage_2::{NorgBlock, ParagraphSegmentToken, ParagraphTokenList}; @@ -141,7 +142,7 @@ fn paragraph_parser_opener_candidates_and_links() -> impl Parser< .at_least(1), ) .then_ignore(just(ParagraphSegmentToken::Special('`'))) - .map(|content| ParagraphSegment::InlineVerbatim(content)); + .map(ParagraphSegment::InlineVerbatim); let anchor = just(ParagraphSegmentToken::Special('[')) .ignore_then( @@ -717,7 +718,7 @@ pub fn stage_3( NorgASTFlat::VerbatimRangedTag { name: stringify_tokens_and_split(name), parameters: parameters.unwrap_or_default().into_iter().map(|parameter| parameter.into_iter().map_into::().collect()).collect(), - content: content.into_iter().map_into::().collect(), + content: dedent(content.into_iter().map_into::().collect::().as_str()), } }, };