Skip to content

Add missing tags #61

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Joeoc2001 wants to merge 3 commits into
base: main
Choose a base branch
from
Open

Add missing tags #61

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
cb37d55
Add prec_dynamic
Joeoc2001 Mar 13, 2025
3d0f8f3
Add immediate tokens
Joeoc2001 Mar 13, 2025
2fb9d31
Clippy
Joeoc2001 Mar 13, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -165,9 +165,12 @@ enum SmallDigit {
}
```

### `#[rust_sitter::prec(...)]` / `#[rust_sitter::prec_left(...)]` / `#[rust_sitter::prec_right(...)]`
### `#[rust_sitter::prec(...)]` / `#[rust_sitter::prec_left(...)]` / `#[rust_sitter::prec_right(...)]` / `#[rust_sitter::prec_dynamic(...)]`
This annotation can be used to define a non/left/right-associative operator. This annotation takes a single parameter, which is the precedence level of the operator (higher binds more tightly).

### `[#rust_sitter::immediate]`
Usually, whitespace is optional before each token. This attribute means that the token will only match if there is no whitespace.

### `#[rust_sitter::skip(...)]`
This annotation can be used to define a field that does not correspond to anything in the input string, such as some metadata. This annotation takes a single parameter, which is the value that should be used to populate that field at runtime.

Expand Down
39 changes: 39 additions & 0 deletions macro/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,45 @@ pub fn prec_right(
item
}

#[proc_macro_attribute]
/// This macro is similar to [`prec`], but the given numerical precedence is applied at runtime instead
/// of at parser generation time. This is only necessary when handling a conflict dynamically using
/// [`conflicts`], and when there is a genuine ambiguity: multiple rules correctly
/// match a given piece of code. In that event, Rust-sitter compares the total dynamic precedence
/// associated with each rule, and selects the one with the highest total.
///
/// This is similar to dynamic precedence directives in Bison grammars.
///
/// ## Example
/// ```ignore
/// #[rust_sitter::prec_dynamic(1)]
/// Cons(Box<Expr>, Box<Expr>)
/// ```
pub fn prec_dynamic(
_attr: proc_macro::TokenStream,
item: proc_macro::TokenStream,
) -> proc_macro::TokenStream {
item
}

#[proc_macro_attribute]
/// Usually, whitespace is optional before each token. This attribute means that the token will only match if there is no whitespace.
///
/// ## Example
/// ```ignore
/// struct StringFragment(
/// #[rust_sitter::immediate]
/// #[rust_sitter::leaf(pattern = r"[^"\\]+")]
/// ()
/// );
/// ```
pub fn immediate(
_attr: proc_macro::TokenStream,
item: proc_macro::TokenStream,
) -> proc_macro::TokenStream {
item
}

#[proc_macro_attribute]
/// On `Vec<_>` typed fields, specifies a non-terminal that should be parsed in between the elements.
/// The [`rust_sitter::repeat`] annotation must be used on the field as well.
Expand Down
211 changes: 146 additions & 65 deletions tool/src/expansion.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,126 @@ use rust_sitter_common::*;
use serde_json::{json, Map, Value};
use syn::{parse::Parse, punctuated::Punctuated, *};

struct Precs {
prec_param: Option<Expr>,
prec_left_param: Option<Expr>,
prec_right_param: Option<Expr>,
prec_dynamic_param: Option<Expr>,
immediate: bool,
}

impl Precs {
fn new(attrs: &[Attribute]) -> Self {
let prec_attr = attrs
.iter()
.find(|attr| attr.path() == &syn::parse_quote!(rust_sitter::prec));

let prec_param = prec_attr.and_then(|a| a.parse_args_with(Expr::parse).ok());

let prec_left_attr = attrs
.iter()
.find(|attr| attr.path() == &syn::parse_quote!(rust_sitter::prec_left));

let prec_left_param = prec_left_attr.and_then(|a| a.parse_args_with(Expr::parse).ok());

let prec_right_attr = attrs
.iter()
.find(|attr| attr.path() == &syn::parse_quote!(rust_sitter::prec_right));

let prec_right_param = prec_right_attr.and_then(|a| a.parse_args_with(Expr::parse).ok());

let prec_dynamic_attr = attrs
.iter()
.find(|attr| attr.path() == &syn::parse_quote!(rust_sitter::prec_dynamic));

let prec_dynamic_param =
prec_dynamic_attr.and_then(|a| a.parse_args_with(Expr::parse).ok());

let immediate_attr = attrs
.iter()
.find(|attr| attr.path() == &syn::parse_quote!(rust_sitter::immediate));

Self {
prec_param,
prec_left_param,
prec_right_param,
prec_dynamic_param,
immediate: immediate_attr.is_some(),
}
}

fn apply(self, rule: serde_json::Value) -> serde_json::Value {
let Self {
prec_param,
prec_left_param,
prec_right_param,
prec_dynamic_param,
immediate,
} = self;

let rule = if let Some(Expr::Lit(lit)) = prec_param {
if prec_left_param.is_some() || prec_right_param.is_some() {
panic!("only one of prec, prec_left, and prec_right can be specified");
}

if let Lit::Int(i) = &lit.lit {
json!({
"type": "PREC",
"value": i.base10_parse::<u32>().unwrap(),
"content": rule
})
} else {
panic!("Expected integer literal for precedence");
}
} else if let Some(Expr::Lit(lit)) = prec_left_param {
if prec_right_param.is_some() {
panic!("only one of prec, prec_left, and prec_right can be specified");
}

if let Lit::Int(i) = &lit.lit {
json!({
"type": "PREC_LEFT",
"value": i.base10_parse::<u32>().unwrap(),
"content": rule
})
} else {
panic!("Expected integer literal for precedence");
}
} else if let Some(Expr::Lit(lit)) = prec_right_param {
if let Lit::Int(i) = &lit.lit {
json!({
"type": "PREC_RIGHT",
"value": i.base10_parse::<u32>().unwrap(),
"content": rule
})
} else {
panic!("Expected integer literal for precedence");
}
} else if let Some(Expr::Lit(lit)) = prec_dynamic_param {
if let Lit::Int(i) = &lit.lit {
json!({
"type": "PREC_DYNAMIC",
"value": i.base10_parse::<u32>().unwrap(),
"content": rule
})
} else {
panic!("Expected integer literal for dynamic precedence");
}
} else {
rule
};

if immediate {
json!({
"type": "IMMEDIATE_TOKEN",
"content": rule
})
} else {
rule
}
}
}

fn gen_field(
path: String,
leaf_type: Type,
Expand All @@ -26,6 +146,13 @@ fn gen_field(
*word_rule = Some(path.clone());
}

let precs = Precs::new(&leaf_attrs);
if precs.prec_left_param.is_some() || precs.prec_right_param.is_some() {
panic!(
"The attributes `prec_left` and `prec_right` cannot be applied to a non-struct type"
);
}

let leaf_params = leaf_attr.and_then(|a| {
a.parse_args_with(Punctuated::<NameValueExpr, Token![,]>::parse_terminated)
.ok()
Expand Down Expand Up @@ -55,10 +182,10 @@ fn gen_field(
if let Lit::Str(s) = &lit.lit {
out.insert(
path.clone(),
json!({
precs.apply(json!({
"type": "PATTERN",
"value": s.value(),
}),
})),
);

(
Expand All @@ -75,10 +202,10 @@ fn gen_field(
if let Lit::Str(s) = &lit.lit {
out.insert(
path.clone(),
json!({
precs.apply(json!({
"type": "STRING",
"value": s.value(),
}),
})),
);

(
Expand All @@ -103,10 +230,10 @@ fn gen_field(
};

(
json!({
precs.apply(json!({
"type": "SYMBOL",
"name": symbol_name,
}),
})),
false,
)
}
Expand Down Expand Up @@ -216,6 +343,8 @@ fn gen_field(
})
};

let vec_contents = precs.apply(vec_contents);

let contents_ident = format!("{path}_vec_contents");
out.insert(contents_ident.clone(), vec_contents);

Expand All @@ -235,7 +364,7 @@ fn gen_field(
panic!("Option<Option<_>> is not supported");
}

(field_json, true)
(precs.apply(field_json), true)
}
}

Expand Down Expand Up @@ -304,23 +433,7 @@ fn gen_struct_or_variant(
})
.collect::<Vec<Value>>();

let prec_attr = attrs
.iter()
.find(|attr| attr.path() == &syn::parse_quote!(rust_sitter::prec));

let prec_param = prec_attr.and_then(|a| a.parse_args_with(Expr::parse).ok());

let prec_left_attr = attrs
.iter()
.find(|attr| attr.path() == &syn::parse_quote!(rust_sitter::prec_left));

let prec_left_param = prec_left_attr.and_then(|a| a.parse_args_with(Expr::parse).ok());

let prec_right_attr = attrs
.iter()
.find(|attr| attr.path() == &syn::parse_quote!(rust_sitter::prec_right));

let prec_right_param = prec_right_attr.and_then(|a| a.parse_args_with(Expr::parse).ok());
let precs = Precs::new(&attrs);

let base_rule = match fields {
Fields::Unit => {
Expand All @@ -343,47 +456,7 @@ fn gen_struct_or_variant(
}),
};

let rule = if let Some(Expr::Lit(lit)) = prec_param {
if prec_left_attr.is_some() || prec_right_attr.is_some() {
panic!("only one of prec, prec_left, and prec_right can be specified");
}

if let Lit::Int(i) = &lit.lit {
json!({
"type": "PREC",
"value": i.base10_parse::<u32>().unwrap(),
"content": base_rule
})
} else {
panic!("Expected integer literal for precedence");
}
} else if let Some(Expr::Lit(lit)) = prec_left_param {
if prec_right_attr.is_some() {
panic!("only one of prec, prec_left, and prec_right can be specified");
}

if let Lit::Int(i) = &lit.lit {
json!({
"type": "PREC_LEFT",
"value": i.base10_parse::<u32>().unwrap(),
"content": base_rule
})
} else {
panic!("Expected integer literal for precedence");
}
} else if let Some(Expr::Lit(lit)) = prec_right_param {
if let Lit::Int(i) = &lit.lit {
json!({
"type": "PREC_RIGHT",
"value": i.base10_parse::<u32>().unwrap(),
"content": base_rule
})
} else {
panic!("Expected integer literal for precedence");
}
} else {
base_rule
};
let rule = precs.apply(base_rule);

out.insert(path, rule);
}
Expand Down Expand Up @@ -466,6 +539,14 @@ pub fn generate_grammar(module: &ItemMod) -> Value {
"members": members
});

let precs = Precs::new(&e.attrs);
if precs.prec_left_param.is_some() || precs.prec_right_param.is_some() {
panic!(
"The attributes `prec_left` and `prec_right` cannot be applied directly to an enum"
);
}
let rule = precs.apply(rule);

rules_map.insert(e.ident.to_string(), rule);

(e.ident.to_string(), e.attrs.clone())
Expand Down
Loading