Add feature to add spans tracking source positions in AST nodes

.gitattributes

@@ -1,4 +1,6 @@
 *.ftl eol=lf
 
-fluent-syntax/tests/fixtures/crlf.ftl eol=crlf
 fluent-syntax/tests/fixtures/cr.ftl eol=cr
+fluent-syntax/tests/fixtures/crlf.ftl eol=crlf
+fluent-syntax/tests/fixtures/formatted/cr.ftl eol=cr
+fluent-syntax/tests/fixtures/formatted/junked/cr.ftl eol=cr

fluent-bundle/CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## Unreleased
   - Implement NUMBER builtin
+  - Adapt for new optional source positions span feature in syntax parser
   - Improve examples
   - Refactor to remove unnecessary named lifetimes
   - Cleanup docs

fluent-bundle/src/resolver/errors.rs

@@ -31,7 +31,7 @@ where
             InlineExpression::FunctionReference { id, .. } => Self::Function {
                 id: id.name.to_string(),
             },
-            InlineExpression::MessageReference { id, attribute } => Self::Message {
+            InlineExpression::MessageReference { id, attribute, .. } => Self::Message {
                 id: id.name.to_string(),
                 attribute: attribute.as_ref().map(|i| i.name.to_string()),
             },

fluent-bundle/src/resolver/expression.rs

@@ -23,15 +23,17 @@ impl<'bundle> WriteValue<'bundle> for ast::Expression<&'bundle str> {
         M: MemoizerKind,
     {
         match self {
-            Self::Inline(exp) => exp.write(w, scope),
-            Self::Select { selector, variants } => {
+            Self::Inline(exp, ..) => exp.write(w, scope),
+            Self::Select {
+                selector, variants, ..
+            } => {
                 let selector = selector.resolve(scope);
                 match selector {
                     FluentValue::String(_) | FluentValue::Number(_) => {
                         for variant in variants {
                             let key = match variant.key {
-                                ast::VariantKey::Identifier { name } => name.into(),
-                                ast::VariantKey::NumberLiteral { value } => {
+                                ast::VariantKey::Identifier { name, .. } => name.into(),
+                                ast::VariantKey::NumberLiteral { value, .. } => {
                                     FluentValue::try_number(value)
                                 }
                             };
@@ -59,7 +61,7 @@ impl<'bundle> WriteValue<'bundle> for ast::Expression<&'bundle str> {
         W: fmt::Write,
     {
         match self {
-            Self::Inline(exp) => exp.write_error(w),
+            Self::Inline(exp, ..) => exp.write_error(w),
             Self::Select { selector, .. } => selector.write_error(w),
         }
     }

fluent-bundle/src/resolver/inline_expression.rs

@@ -24,8 +24,8 @@ impl<'bundle> WriteValue<'bundle> for ast::InlineExpression<&'bundle str> {
         M: MemoizerKind,
     {
         match self {
-            Self::StringLiteral { value } => unescape_unicode(w, value),
-            Self::MessageReference { id, attribute } => {
+            Self::StringLiteral { value, .. } => unescape_unicode(w, value),
+            Self::MessageReference { id, attribute, .. } => {
                 if let Some(msg) = scope.bundle.get_entry_message(id.name) {
                     if let Some(attr) = attribute {
                         msg.attributes
@@ -53,11 +53,12 @@ impl<'bundle> WriteValue<'bundle> for ast::InlineExpression<&'bundle str> {
                     scope.write_ref_error(w, self)
                 }
             }
-            Self::NumberLiteral { value } => FluentValue::try_number(value).write(w, scope),
+            Self::NumberLiteral { value, .. } => FluentValue::try_number(value).write(w, scope),
             Self::TermReference {
                 id,
                 attribute,
                 arguments,
+                ..
             } => {
                 let (_, resolved_named_args) = scope.get_arguments(arguments.as_ref());
 
@@ -82,7 +83,7 @@ impl<'bundle> WriteValue<'bundle> for ast::InlineExpression<&'bundle str> {
                 scope.local_args = None;
                 result
             }
-            Self::FunctionReference { id, arguments } => {
+            Self::FunctionReference { id, arguments, .. } => {
                 let (resolved_positional_args, resolved_named_args) =
                     scope.get_arguments(Some(arguments));
 
@@ -99,7 +100,7 @@ impl<'bundle> WriteValue<'bundle> for ast::InlineExpression<&'bundle str> {
                     scope.write_ref_error(w, self)
                 }
             }
-            Self::VariableReference { id } => {
+            Self::VariableReference { id, .. } => {
                 let args = scope.local_args.as_ref().or(scope.args);
 
                 if let Some(arg) = args.and_then(|args| args.get(id.name)) {
@@ -113,7 +114,7 @@ impl<'bundle> WriteValue<'bundle> for ast::InlineExpression<&'bundle str> {
                     w.write_char('}')
                 }
             }
-            Self::Placeable { expression } => expression.write(w, scope),
+            Self::Placeable { expression, .. } => expression.write(w, scope),
         }
     }
 
@@ -125,10 +126,12 @@ impl<'bundle> WriteValue<'bundle> for ast::InlineExpression<&'bundle str> {
             Self::MessageReference {
                 id,
                 attribute: Some(attribute),
+                ..
             } => write!(w, "{}.{}", id.name, attribute.name),
             Self::MessageReference {
                 id,
                 attribute: None,
+                ..
             } => w.write_str(id.name),
             Self::TermReference {
                 id,
@@ -141,7 +144,7 @@ impl<'bundle> WriteValue<'bundle> for ast::InlineExpression<&'bundle str> {
                 ..
             } => write!(w, "-{}", id.name),
             Self::FunctionReference { id, .. } => write!(w, "{}()", id.name),
-            Self::VariableReference { id } => write!(w, "${}", id.name),
+            Self::VariableReference { id, .. } => write!(w, "${}", id.name),
             _ => unreachable!(),
         }
     }
@@ -157,9 +160,9 @@ impl<'bundle> ResolveValue<'bundle> for ast::InlineExpression<&'bundle str> {
         M: MemoizerKind,
     {
         match self {
-            Self::StringLiteral { value } => unescape_unicode_to_string(value).into(),
-            Self::NumberLiteral { value } => FluentValue::try_number(value),
-            Self::VariableReference { id } => {
+            Self::StringLiteral { value, .. } => unescape_unicode_to_string(value).into(),
+            Self::NumberLiteral { value, .. } => FluentValue::try_number(value),
+            Self::VariableReference { id, .. } => {
                 if let Some(local_args) = &scope.local_args {
                     if let Some(arg) = local_args.get(id.name) {
                         return arg.clone();
@@ -173,7 +176,7 @@ impl<'bundle> ResolveValue<'bundle> for ast::InlineExpression<&'bundle str> {
                 }
                 FluentValue::Error
             }
-            Self::FunctionReference { id, arguments } => {
+            Self::FunctionReference { id, arguments, .. } => {
                 let (resolved_positional_args, resolved_named_args) =
                     scope.get_arguments(Some(arguments));
 

fluent-bundle/src/resolver/pattern.rs

@@ -32,14 +32,14 @@ impl<'bundle> WriteValue<'bundle> for ast::Pattern<&'bundle str> {
             }
 
             match elem {
-                ast::PatternElement::TextElement { value } => {
+                ast::PatternElement::TextElement { value, .. } => {
                     if let Some(ref transform) = scope.bundle.transform {
                         w.write_str(&transform(value))?;
                     } else {
                         w.write_str(value)?;
                     }
                 }
-                ast::PatternElement::Placeable { ref expression } => {
+                ast::PatternElement::Placeable { ref expression, .. } => {
                     scope.placeables += 1;
                     if scope.placeables > MAX_PLACEABLES {
                         scope.dirty = true;
@@ -51,13 +51,16 @@ impl<'bundle> WriteValue<'bundle> for ast::Pattern<&'bundle str> {
                         && len > 1
                         && !matches!(
                             expression,
-                            ast::Expression::Inline(ast::InlineExpression::MessageReference { .. },)
-                                | ast::Expression::Inline(
-                                    ast::InlineExpression::TermReference { .. },
-                                )
-                                | ast::Expression::Inline(
-                                    ast::InlineExpression::StringLiteral { .. },
-                                )
+                            ast::Expression::Inline(
+                                ast::InlineExpression::MessageReference { .. },
+                                ..
+                            ) | ast::Expression::Inline(
+                                ast::InlineExpression::TermReference { .. },
+                                ..
+                            ) | ast::Expression::Inline(
+                                ast::InlineExpression::StringLiteral { .. },
+                                ..
+                            )
                         );
                     if needs_isolation {
                         w.write_char('\u{2068}')?;
@@ -92,7 +95,7 @@ impl<'bundle> ResolveValue<'bundle> for ast::Pattern<&'bundle str> {
         let len = self.elements.len();
 
         if len == 1 {
-            if let ast::PatternElement::TextElement { value } = self.elements[0] {
+            if let ast::PatternElement::TextElement { value, .. } = self.elements[0] {
                 return scope
                     .bundle
                     .transform

fluent-bundle/src/resolver/scope.rs

@@ -124,7 +124,10 @@ impl<'bundle, 'ast, 'args, 'errors, R, M> Scope<'bundle, 'ast, 'args, 'errors, R
         R: Borrow<FluentResource>,
         M: MemoizerKind,
     {
-        if let Some(ast::CallArguments { positional, named }) = arguments {
+        if let Some(ast::CallArguments {
+            positional, named, ..
+        }) = arguments
+        {
             let positional = positional.iter().map(|expr| expr.resolve(self)).collect();
 
             let named = named

fluent-syntax/CHANGELOG.md

@@ -1,6 +1,7 @@
 # Changelog
 
 ## Unreleased
+  - Add `spans` feature to track source positions in the AST
   - Add module `serializer`
   - De-ambiguate dependencies vs. features
   - Cleanup docs

fluent-syntax/Cargo.toml

@@ -38,6 +38,7 @@ glob = "0.3"
 
 [features]
 default = []
+spans = []
 serde = ["dep:serde"]
 json = ["serde", "dep:serde_json"]
 all-benchmarks = []
@@ -63,3 +64,6 @@ required-features = ["json"]
 name = "parser_fixtures"
 path = "tests/parser_fixtures.rs"
 required-features = ["json"]
+
+[lib]
+doctest = false

fluent-syntax/src/ast/helper.rs

@@ -1,25 +1,49 @@
+use super::Comment;
+#[cfg(feature = "spans")]
+use super::Span;
 #[cfg(feature = "serde")]
 use serde::{Deserialize, Serialize};
 
-use super::Comment;
 // This is a helper struct used to properly deserialize referential
 // JSON comments which are single continuous String, into a vec of
 // content slices.
 #[derive(Clone, Debug, Eq, PartialEq)]
 #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
 #[cfg_attr(feature = "serde", serde(untagged))]
 pub enum CommentDef<S> {
-    Single { content: S },
-    Multi { content: Vec<S> },
+    Single {
+        content: S,
+        #[cfg(feature = "spans")]
+        span: Span,
+    },
+    Multi {
+        content: Vec<S>,
+        #[cfg(feature = "spans")]
+        span: Span,
+    },
 }
 
 impl<S> From<CommentDef<S>> for Comment<S> {
     fn from(input: CommentDef<S>) -> Self {
         match input {
-            CommentDef::Single { content } => Self {
+            CommentDef::Single {
+                content,
+                #[cfg(feature = "spans")]
+                span,
+            } => Self {
                 content: vec![content],
+                #[cfg(feature = "spans")]
+                span,
+            },
+            CommentDef::Multi {
+                content,
+                #[cfg(feature = "spans")]
+                span,
+            } => Self {
+                content,
+                #[cfg(feature = "spans")]
+                span,
             },
-            CommentDef::Multi { content } => Self { content },
         }
     }
 }