Skip to content

Update wikitext parsing functionality #3

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
philpax merged 2 commits into from
Nov 18, 2025
Merged

Update wikitext parsing functionality #3

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4dcec75
Update wikitext_simplified to latest version and fix Spanned wrapper
claude Nov 18, 2025
99b4c6b
Refactor: Extract empty_spanned helper to common util module
claude Nov 18, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions Cargo.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

47 changes: 29 additions & 18 deletions src/main.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,16 @@ use std::{
};

use template::{TemplateToInstantiate, Templates};
use wikitext_simplified::{WikitextSimplifiedNode, wikitext_util::parse_wiki_text_2};
use wikitext_simplified::{WikitextSimplifiedNode, Spanned, wikitext_util::parse_wiki_text_2};

mod page_context;
use page_context::PageContext;

mod syntax;
mod template;
mod util;

use util::empty_spanned;

const WIKI_DIRECTORY: &str = "wiki";

Expand Down Expand Up @@ -296,7 +299,8 @@ fn generate_wiki_folder(
.map(|s| s.to_string()),
);

let document = if let [WikitextSimplifiedNode::Redirect { target }] = simplified.as_slice()
let document = if let [node] = simplified.as_slice()
&& let WikitextSimplifiedNode::Redirect { target } = &node.value
{
redirect(&page_title_to_route_path(target).url_path())
} else {
Expand All @@ -323,7 +327,7 @@ fn generate_wiki_folder(
layout(
&page_context.title,
paxhtml::Element::from_iter(simplified.iter().map(|node| {
convert_wikitext_to_html(templates, pwt_configuration, node, &page_context)
convert_wikitext_to_html(templates, pwt_configuration, &node.value, &page_context)
})),
)
};
Expand Down Expand Up @@ -437,7 +441,7 @@ fn convert_wikitext_to_html(
let attributes = templates.instantiate(
pwt_configuration,
TemplateToInstantiate::Node(WikitextSimplifiedNode::Fragment {
children: attributes.to_vec(),
children: attributes.iter().map(|n| empty_spanned(n.clone())).collect(),
}),
&[],
page_context,
Expand All @@ -455,7 +459,7 @@ fn convert_wikitext_to_html(
let merged_text = attributes
.iter()
.filter_map(|node| {
if let WSN::Text { text } = node {
if let WSN::Text { text } = &node.value {
Some(text.as_str())
} else {
None
Expand All @@ -478,29 +482,30 @@ fn convert_wikitext_to_html(
pwt_configuration: &parse_wiki_text_2::Configuration,
page_context: &PageContext,
attributes_context: &str,
attributes: &Option<Vec<WSN>>,
attributes: &Option<Vec<Spanned<WSN>>>,
) -> Vec<paxhtml::Attribute> {
attributes
.as_deref()
.as_ref()
.map(|attributes| {
let unwrapped: Vec<WSN> = attributes.iter().map(|s| s.value.clone()).collect();
parse_attributes_from_wsn(
templates,
pwt_configuration,
page_context,
attributes_context,
attributes,
&unwrapped,
)
})
.unwrap_or_default()
}

let convert_children = |templates: &mut Templates, children: &[WikitextSimplifiedNode]| {
let convert_children = |templates: &mut Templates, children: &[Spanned<WikitextSimplifiedNode>]| {
paxhtml::Element::from_iter(
children
.iter()
.skip_while(|node| matches!(node, WSN::ParagraphBreak | WSN::Newline))
.skip_while(|node| matches!(node.value, WSN::ParagraphBreak | WSN::Newline))
.map(|node| {
convert_wikitext_to_html(templates, pwt_configuration, node, page_context)
convert_wikitext_to_html(templates, pwt_configuration, &node.value, page_context)
}),
)
};
Expand Down Expand Up @@ -591,7 +596,9 @@ fn convert_wikitext_to_html(
};

// Get the code text
let code = if let [WSN::Text { text }] = children.as_slice() {
let code = if let [node] = children.as_slice()
&& let WSN::Text { text } = &node.value
{
text.trim()
} else {
// If not simple text, fall back to plain rendering
Expand Down Expand Up @@ -642,13 +649,15 @@ fn convert_wikitext_to_html(
let instantiated = templates.instantiate(
pwt_configuration,
TemplateToInstantiate::Node(WikitextSimplifiedNode::Fragment {
children: attributes.to_vec(),
children: attributes.iter().map(|n| empty_spanned(n.value.clone())).collect(),
}),
&[],
page_context,
);
if let WSN::Fragment { children } = instantiated {
if let Some(WSN::Text { text }) = children.first() {
if let Some(node) = children.first()
&& let WSN::Text { text } = &node.value
{
text.contains("class=")
} else {
false
Expand All @@ -662,18 +671,19 @@ fn convert_wikitext_to_html(

if !has_class_attr {
// Add Tailwind table classes
modified_attributes.push(WSN::Text {
modified_attributes.push(empty_spanned(WSN::Text {
text: " class=\"min-w-full divide-y divide-gray-200 border border-gray-300\""
.to_string(),
});
}));
}

let unwrapped_attributes: Vec<WSN> = modified_attributes.iter().map(|s| s.value.clone()).collect();
let attributes = parse_attributes_from_wsn(
templates,
pwt_configuration,
page_context,
"main",
&modified_attributes,
&unwrapped_attributes,
);
html! {
<table {attributes}>
Expand Down Expand Up @@ -703,12 +713,13 @@ fn convert_wikitext_to_html(
.iter()
.enumerate()
.map(|(idx, row)| {
let unwrapped_row_attrs: Vec<WSN> = row.attributes.iter().map(|s| s.value.clone()).collect();
let attributes = parse_attributes_from_wsn(
templates,
pwt_configuration,
page_context,
"row",
&row.attributes,
&unwrapped_row_attrs,
);
let row_class = if idx % 2 == 0 { "bg-white" } else { "bg-gray-50" };
html! {
Expand Down
16 changes: 9 additions & 7 deletions src/template.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ use std::{
use wikitext_simplified::{TemplateParameter, WikitextSimplifiedNode, parse_wiki_text_2};

use crate::page_context::PageContext;
use crate::util::empty_spanned;

/// Trait for loading wikitext template files
pub trait TemplateLoader {
Expand Down Expand Up @@ -137,7 +138,7 @@ impl<'a> Templates<'a> {
cell.content = children;
}
other => {
cell.content = vec![other];
cell.content = vec![empty_spanned(other)];
}
}
}
Expand All @@ -146,7 +147,7 @@ impl<'a> Templates<'a> {
}
WSN::Fragment { children } => {
for child in children {
self.reparse_table_cells(child, pwt_configuration, page_context);
self.reparse_table_cells(&mut child.value, pwt_configuration, page_context);
}
}
_ => {}
Expand Down Expand Up @@ -228,7 +229,7 @@ impl<'a> Templates<'a> {
// Flatten single-child fragments to avoid nested structures
match result {
WSN::Fragment { children } if children.len() == 1 => {
children.into_iter().next().unwrap()
children.into_iter().next().unwrap().value
}
_ => result,
}
Expand Down Expand Up @@ -400,10 +401,11 @@ mod tests {
// Verify the result is a table (possibly wrapped in a Fragment)
let table_node = match &result {
WikitextSimplifiedNode::Table { .. } => &result,
WikitextSimplifiedNode::Fragment { children } => children
WikitextSimplifiedNode::Fragment { children } => &children
.iter()
.find(|node| matches!(node, WikitextSimplifiedNode::Table { .. }))
.expect("Fragment should contain a Table node"),
.find(|node| matches!(node.value, WikitextSimplifiedNode::Table { .. }))
.expect("Fragment should contain a Table node")
.value,
_ => panic!(
"Expected Table or Fragment with Table node, got {:?}",
result
Expand Down Expand Up @@ -496,7 +498,7 @@ mod tests {
assert!(
children
.iter()
.any(|node| matches!(node, WikitextSimplifiedNode::Bold { .. })),
.any(|node| matches!(node.value, WikitextSimplifiedNode::Bold { .. })),
"Template should be reparsed into Bold node through wikitext roundtrip"
);
}
Expand Down
9 changes: 9 additions & 0 deletions src/util.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
use wikitext_simplified::{Span, Spanned};

/// Helper to create a Spanned node with a default (empty) span
pub fn empty_spanned<T>(value: T) -> Spanned<T> {
Spanned {
value,
span: Span { start: 0, end: 0 },
}
}