split language sections into LexicalContexts (#1520)

First of two PRs, splitting the langauge sections into different
contexts, each with their own `Identifier`:

- Solidity: main context
- Pragma: since lexing version specifiers conflict with number literals
in Solidity/Yul
- Yul: since lexing identifiers conflicts with Solidity's own
identifiers, as they can contain dots.

The next PR will add validation to actually enforce boundaries between
them, and make sure each context nonterminals can only reference
terminals in the same context.

Author: OmarTawfik

crates/codegen-v2/parser/src/lexer/builder.rs

@@ -49,11 +49,11 @@ impl LexerModelBuilder {
         // For now, we just collect trivia and duplicate them into all contexts:
         let mut common_trivia = Vec::<Lexeme>::new();
 
-        for topic in language.topics() {
-            let context_name = topic.lexical_context.to_string();
-            let context = non_trivia.entry(context_name).or_default();
+        for context in &language.contexts {
+            let context_name = context.name.to_string();
+            let entry = non_trivia.entry(context_name).or_default();
 
-            for item in &topic.items {
+            for item in context.items() {
                 match item {
                     Item::Struct { .. } => {}
                     Item::Enum { .. } => {}
@@ -62,8 +62,8 @@ impl LexerModelBuilder {
                     Item::Precedence { .. } => {}
 
                     Item::Trivia { item } => common_trivia.push(self.convert_trivia(item)),
-                    Item::Keyword { item } => context.extend(self.convert_keyword(item)),
-                    Item::Token { item } => context.extend(self.convert_token(item)),
+                    Item::Keyword { item } => entry.extend(self.convert_keyword(item)),
+                    Item::Token { item } => entry.extend(self.convert_token(item)),
 
                     Item::Fragment { .. } => {}
                 }

crates/language-v2/definition/src/compiler/analysis/mod.rs

@@ -66,8 +66,9 @@ impl Analysis {
 
 impl SpannedLanguage {
     fn items(&self) -> impl Iterator<Item = &SpannedItem> {
-        self.sections
+        self.contexts
             .iter()
+            .flat_map(|context| &context.sections)
             .flat_map(|section| &section.topics)
             .flat_map(|topic| &topic.items)
     }

crates/language-v2/definition/src/compiler/analysis/p2_version_specifiers/mod.rs

@@ -168,7 +168,6 @@ fn check_fields(analysis: &mut Analysis, fields: &IndexMap<Spanned<Identifier>,
 fn check_keyword(analysis: &mut Analysis, item: &SpannedKeywordItem) {
     let SpannedKeywordItem {
         name: _,
-        identifier: _,
         enabled,
         definitions,
     } = item;

crates/language-v2/definition/src/compiler/analysis/p3_references/mod.rs

@@ -296,15 +296,12 @@ fn check_trivia(analysis: &mut Analysis, item: &SpannedTriviaItem, enablement: &
 
 fn check_keyword(analysis: &mut Analysis, item: &SpannedKeywordItem, enablement: &VersionSet) {
     let SpannedKeywordItem {
-        name,
-        identifier,
+        name: _,
         enabled,
         definitions: _,
     } = item;
 
-    let enablement = update_enablement(analysis, enablement, enabled.as_ref());
-
-    check_reference(analysis, Some(name), identifier, &enablement, &[Token]);
+    let _ = update_enablement(analysis, enablement, enabled.as_ref());
 }
 
 fn check_token(analysis: &mut Analysis, item: &SpannedTokenItem, enablement: &VersionSet) {

crates/language-v2/definition/src/model/manifest.rs

@@ -27,22 +27,17 @@ pub struct Language {
     /// The supported versions of the language
     pub versions: IndexSet<Version>,
 
-    /// The sections of the language
-    pub sections: Vec<Section>,
+    /// The lexical contexts of the language, splitting grammar elements based on which lexer can recognize their terminals.
+    pub contexts: Vec<LexicalContext>,
 
     /// The built-in contexts
     pub built_ins: Vec<BuiltInContext>,
 }
 
 impl Language {
-    /// Returns every topic in the language definition (across all sections).
-    pub fn topics(&self) -> impl Iterator<Item = &Topic> {
-        self.sections.iter().flat_map(|section| &section.topics)
-    }
-
-    /// Returns every item in the language definition (across all sections and topics).
+    /// Returns every item in the language definition (across all contexts).
     pub fn items(&self) -> impl Iterator<Item = &Item> {
-        self.topics().flat_map(|topic| &topic.items)
+        self.contexts.iter().flat_map(|context| context.items())
     }
 
     /// Collects all versions that change the language in a breaking way.
@@ -168,6 +163,21 @@ impl Language {
 }
 
 /// A section is a named container for topics, used for organizing the large grammar definition in user documentation.  
+#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
+#[derive_spanned_type(Clone, Debug, ParseInputTokens, WriteOutputTokens)]
+pub struct LexicalContext {
+    pub name: Identifier,
+    pub identifier_token: Option<Identifier>,
+    pub sections: Vec<Section>,
+}
+
+impl LexicalContext {
+    /// Returns every item in that context (across all sections).
+    pub fn items(&self) -> impl Iterator<Item = &Item> {
+        self.sections.iter().flat_map(|section| section.items())
+    }
+}
+
 #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
 #[derive_spanned_type(Clone, Debug, ParseInputTokens, WriteOutputTokens)]
 pub struct Section {
@@ -187,7 +197,6 @@ impl Section {
 #[derive_spanned_type(Clone, Debug, ParseInputTokens, WriteOutputTokens)]
 pub struct Topic {
     pub title: String,
-    pub lexical_context: Identifier,
     pub items: Vec<Item>,
 }
 

crates/language-v2/definition/src/model/terminals/keyword.rs

@@ -8,7 +8,6 @@ use crate::model::{Identifier, Scanner, VersionSpecifier};
 #[derive_spanned_type(Clone, Debug, ParseInputTokens, WriteOutputTokens)]
 pub struct KeywordItem {
     pub name: Identifier,
-    pub identifier: Identifier,
 
     #[serde(skip_serializing_if = "Option::is_none")]
     pub enabled: Option<VersionSpecifier>,

crates/language-v2/internal_macros/src/derive/spanned.rs

@@ -127,6 +127,7 @@ fn get_spanned_type(input: Type) -> Type {
         | "KeywordDefinition"
         | "KeywordItem"
         | "KeywordValue"
+        | "LexicalContext"
         | "PrecedenceExpression"
         | "PrecedenceItem"
         | "PrecedenceOperator"

crates/language-v2/tests/src/fail/p0_parsing/duplicate_map_key/test.rs

@@ -6,22 +6,24 @@ language_v2_macros::compile!(Language(
     leading_trivia = Sequence([]),
     trailing_trivia = Sequence([]),
     versions = ["1.0.0", "2.0.0", "3.0.0"],
-    sections = [Section(
-        title = "Section One",
-        topics = [Topic(
-            title = "Topic One",
-            lexical_context = Foo,
-            items = [
-                Struct(
-                    name = Bar,
-                    fields = (
-                        field_1 = Required(Baz),
-                        field_1 = Required(Baz),
-                        field_3 = Required(Baz)
-                    )
-                ),
-                Token(name = Baz, definitions = [TokenDefinition(Atom("baz"))])
-            ]
+    contexts = [LexicalContext(
+        name = Foo,
+        sections = [Section(
+            title = "Section One",
+            topics = [Topic(
+                title = "Topic One",
+                items = [
+                    Struct(
+                        name = Bar,
+                        fields = (
+                            field_1 = Required(Baz),
+                            field_1 = Required(Baz),
+                            field_3 = Required(Baz)
+                        )
+                    ),
+                    Token(name = Baz, definitions = [TokenDefinition(Atom("baz"))])
+                ]
+            )]
         )]
     )],
     built_ins = []

crates/language-v2/tests/src/fail/p0_parsing/duplicate_map_key/test.stderr

@@ -1,5 +1,5 @@
 error: Duplicate map key.
-  --> src/fail/p0_parsing/duplicate_map_key/test.rs:19:25
+  --> src/fail/p0_parsing/duplicate_map_key/test.rs:20:29
    |
-19 |                         field_1 = Required(Baz),
-   |                         ^^^^^^^
+20 | ...                   field_1 = Required(Baz),
+   |                       ^^^^^^^

crates/language-v2/tests/src/fail/p0_parsing/duplicate_set_entry/test.rs

@@ -6,14 +6,16 @@ language_v2_macros::compile!(Language(
     leading_trivia = Sequence([]),
     trailing_trivia = Sequence([]),
     versions = ["1.0.0", "1.0.0", "3.0.0"],
-    sections = [Section(
-        title = "Section One",
-        topics = [Topic(
-            title = "Topic One",
-            lexical_context = Foo,
-            items = [Token(
-                name = Bar,
-                definitions = [TokenDefinition(Atom("bar"))]
+    contexts = [LexicalContext(
+        name = Foo,
+        sections = [Section(
+            title = "Section One",
+            topics = [Topic(
+                title = "Topic One",
+                items = [Token(
+                    name = Bar,
+                    definitions = [TokenDefinition(Atom("bar"))]
+                )]
             )]
         )]
     )],