[v2] move terminal versioning to parent items instead of child definitions (#1461)

- Promoted `TokenDefinition::enabled` to `TokenItem::enabled`, as the
new lexer doesn't version individual definitions (all of them are
matched in all versions). We can decide on whether to keep or remove the
additional `TokenDefinition` structure after we consider how to handle
post-lexing processing (e.g. `DecimalLiteral` and `YulIdentifier`
version breaks).
- Promoted `KeywordDefinition::enabled` to `KeywordItem::enabled`, as
the new lexer doesn't version individual definitions (all of them are
matched in all versions). We can decide on whether to keep or remove the
additional `KeywordDefinition` structure after we consider how to handle
variations in `reserved` status between them (mainly the four `*fixed*`
keywords).

> NOTE: this is a set of PRs to add the lexer v2, to make reviewing
smaller chunks easier:
>
> 1. #1457
> 2. #1458
> 3. #1459
> 4. #1460
> 5. #1461
> 6. #1462

Author: OmarTawfik

crates/language-v2/definition/src/compiler/analysis/p1_definitions.rs

@@ -224,14 +224,10 @@ fn calculate_defined_in(analysis: &mut Analysis, item: &SpannedItem) -> VersionS
             try_add_specifier(&None);
         }
         SpannedItem::Keyword { item } => {
-            for definition in &item.definitions {
-                try_add_specifier(&definition.enabled);
-            }
+            try_add_specifier(&item.enabled);
         }
         SpannedItem::Token { item } => {
-            for definition in &item.definitions {
-                try_add_specifier(&definition.enabled);
-            }
+            try_add_specifier(&item.enabled);
         }
         SpannedItem::Fragment { item } => {
             try_add_specifier(&item.enabled);

crates/language-v2/definition/src/compiler/analysis/p2_version_specifiers.rs

@@ -11,8 +11,8 @@ use crate::model::{
     SpannedEnumItem, SpannedEnumVariant, SpannedField, SpannedFragmentItem, SpannedItem,
     SpannedKeywordDefinition, SpannedKeywordItem, SpannedPrecedenceExpression,
     SpannedPrecedenceItem, SpannedPrecedenceOperator, SpannedPrimaryExpression,
-    SpannedRepeatedItem, SpannedSeparatedItem, SpannedStructItem, SpannedTokenDefinition,
-    SpannedTokenItem, SpannedVersionSpecifier,
+    SpannedRepeatedItem, SpannedSeparatedItem, SpannedStructItem, SpannedTokenItem,
+    SpannedVersionSpecifier,
 };
 
 pub(crate) fn run(analysis: &mut Analysis) {
@@ -169,35 +169,27 @@ fn check_keyword(analysis: &mut Analysis, item: &SpannedKeywordItem) {
     let SpannedKeywordItem {
         name: _,
         identifier: _,
+        enabled,
         definitions,
     } = item;
 
+    check_version_specifier(analysis, enabled.as_ref());
+
     for definition in definitions {
-        let SpannedKeywordDefinition {
-            enabled,
-            reserved,
-            value: _,
-        } = definition;
+        let SpannedKeywordDefinition { reserved, value: _ } = definition;
 
-        check_version_specifier(analysis, enabled.as_ref());
         check_version_specifier(analysis, reserved.as_ref());
     }
 }
 
 fn check_token(analysis: &mut Analysis, item: &SpannedTokenItem) {
     let SpannedTokenItem {
         name: _,
-        definitions,
+        enabled,
+        definitions: _,
     } = item;
 
-    for definition in definitions {
-        let SpannedTokenDefinition {
-            enabled,
-            scanner: _,
-        } = definition;
-
-        check_version_specifier(analysis, enabled.as_ref());
-    }
+    check_version_specifier(analysis, enabled.as_ref());
 }
 
 fn check_fragment(analysis: &mut Analysis, item: &SpannedFragmentItem) {

crates/language-v2/definition/src/compiler/analysis/p3_references.rs

@@ -12,11 +12,10 @@ use crate::model::SpannedItemDiscriminants::{
 use crate::model::{
     Identifier, SpannedBuiltIn, SpannedBuiltInField, SpannedBuiltInFunction, SpannedBuiltInType,
     SpannedEnumItem, SpannedEnumVariant, SpannedField, SpannedFragmentItem, SpannedItem,
-    SpannedKeywordDefinition, SpannedKeywordItem, SpannedPrecedenceExpression,
-    SpannedPrecedenceItem, SpannedPrecedenceOperator, SpannedPrimaryExpression,
-    SpannedRepeatedItem, SpannedScanner, SpannedSeparatedItem, SpannedStructItem,
-    SpannedTokenDefinition, SpannedTokenItem, SpannedTriviaItem, SpannedTriviaParser,
-    SpannedVersionSpecifier,
+    SpannedKeywordItem, SpannedPrecedenceExpression, SpannedPrecedenceItem,
+    SpannedPrecedenceOperator, SpannedPrimaryExpression, SpannedRepeatedItem, SpannedScanner,
+    SpannedSeparatedItem, SpannedStructItem, SpannedTokenDefinition, SpannedTokenItem,
+    SpannedTriviaItem, SpannedTriviaParser, SpannedVersionSpecifier,
 };
 
 pub(crate) fn run(analysis: &mut Analysis) {
@@ -299,29 +298,26 @@ fn check_keyword(analysis: &mut Analysis, item: &SpannedKeywordItem, enablement:
     let SpannedKeywordItem {
         name,
         identifier,
-        definitions,
+        enabled,
+        definitions: _,
     } = item;
 
-    check_reference(analysis, Some(name), identifier, enablement, &[Token]);
-
-    for definition in definitions {
-        let SpannedKeywordDefinition {
-            enabled,
-            reserved: _,
-            value: _,
-        } = definition;
+    let enablement = update_enablement(analysis, enablement, enabled.as_ref());
 
-        let _ = update_enablement(analysis, enablement, enabled.as_ref());
-    }
+    check_reference(analysis, Some(name), identifier, &enablement, &[Token]);
 }
 
 fn check_token(analysis: &mut Analysis, item: &SpannedTokenItem, enablement: &VersionSet) {
-    let SpannedTokenItem { name, definitions } = item;
+    let SpannedTokenItem {
+        name,
+        enabled,
+        definitions,
+    } = item;
 
-    for definition in definitions {
-        let SpannedTokenDefinition { enabled, scanner } = definition;
+    let enablement = update_enablement(analysis, enablement, enabled.as_ref());
 
-        let enablement = update_enablement(analysis, enablement, enabled.as_ref());
+    for definition in definitions {
+        let SpannedTokenDefinition { scanner } = definition;
 
         check_scanner(analysis, Some(name), scanner, &enablement);
     }

crates/language-v2/definition/src/model/manifest.rs

@@ -88,15 +88,13 @@ impl Language {
                     }
                 }
                 Item::Keyword { item } => {
+                    add_spec(&item.enabled);
                     for definition in &item.definitions {
-                        add_spec(&definition.enabled);
                         add_spec(&definition.reserved);
                     }
                 }
                 Item::Token { item } => {
-                    for definition in &item.definitions {
-                        add_spec(&definition.enabled);
-                    }
+                    add_spec(&item.enabled);
                 }
                 Item::Fragment { item } => add_spec(&item.enabled),
                 Item::Trivia { .. } => {}

crates/language-v2/definition/src/model/terminals/keyword.rs

@@ -10,6 +10,9 @@ pub struct KeywordItem {
     pub name: Identifier,
     pub identifier: Identifier,
 
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub enabled: Option<VersionSpecifier>,
+
     pub definitions: Vec<KeywordDefinition>,
 }
 
@@ -24,9 +27,6 @@ impl KeywordItem {
 #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
 #[derive_spanned_type(Clone, Debug, ParseInputTokens, WriteOutputTokens)]
 pub struct KeywordDefinition {
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub enabled: Option<VersionSpecifier>,
-
     /// When the keyword is reserved, i.e. can't be used in other position (e.g. as a name)
     #[serde(skip_serializing_if = "Option::is_none")]
     pub reserved: Option<VersionSpecifier>,

crates/language-v2/definition/src/model/terminals/token.rs

@@ -8,6 +8,9 @@ use crate::model::{Identifier, Scanner, VersionSpecifier};
 pub struct TokenItem {
     pub name: Identifier,
 
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub enabled: Option<VersionSpecifier>,
+
     pub definitions: Vec<TokenDefinition>,
 }
 
@@ -22,9 +25,6 @@ impl TokenItem {
 #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
 #[derive_spanned_type(Clone, Debug, ParseInputTokens, WriteOutputTokens)]
 pub struct TokenDefinition {
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub enabled: Option<VersionSpecifier>,
-
     pub scanner: Scanner,
 }
 

crates/language-v2/tests/src/fail/p0_parsing/duplicate_map_key/test.rs

@@ -21,10 +21,7 @@ language_v2_macros::compile!(Language(
                         field_3 = Required(Baz)
                     )
                 ),
-                Token(
-                    name = Baz,
-                    definitions = [TokenDefinition(scanner = Atom("baz"))]
-                )
+                Token(name = Baz, definitions = [TokenDefinition(Atom("baz"))])
             ]
         )]
     )],

crates/language-v2/tests/src/fail/p0_parsing/duplicate_set_entry/test.rs

@@ -14,7 +14,7 @@ language_v2_macros::compile!(Language(
             lexical_context = Foo,
             items = [Token(
                 name = Bar,
-                definitions = [TokenDefinition(scanner = Atom("bar"))]
+                definitions = [TokenDefinition(Atom("bar"))]
             )]
         )]
     )],

crates/language-v2/tests/src/fail/p0_parsing/empty_string/test.rs

@@ -12,10 +12,7 @@ language_v2_macros::compile!(Language(
         topics = [Topic(
             title = "Topic One",
             lexical_context = Foo,
-            items = [Token(
-                name = Bar,
-                definitions = [TokenDefinition(scanner = Atom(""))]
-            )]
+            items = [Token(name = Bar, definitions = [TokenDefinition(Atom(""))])]
         )]
     )],
     built_ins = []

crates/language-v2/tests/src/fail/p0_parsing/empty_string/test.stderr

@@ -1,5 +1,5 @@
 error: Expected a non-empty string.
-  --> src/fail/p0_parsing/empty_string/test.rs:17:63
+  --> src/fail/p0_parsing/empty_string/test.rs:15:76
    |
-17 |                 definitions = [TokenDefinition(scanner = Atom(""))]
-   |                                                               ^^
+15 |             items = [Token(name = Bar, definitions = [TokenDefinition(Atom(""))])]
+   |                                                                            ^^