make template checker script more robust

scripts/validate_xls_template.py

@@ -30,6 +30,8 @@
 from dataclasses import dataclass
 from pathlib import Path
 from typing import List, Optional, Tuple
+import urllib.request
+import urllib.error
 
 from markdown_it import MarkdownIt
 from xls_parser import extract_xls_metadata
@@ -86,6 +88,10 @@ class XLSTemplateValidator:
     # Valid category values
     VALID_CATEGORIES = ["Amendment", "System", "Ecosystem", "Meta"]
 
+    # Note: Ledger entry and transaction existence is now determined dynamically
+    # by checking if the documentation exists on xrpl.org rather than
+    # maintaining hardcoded lists.
+
     # Amendment template section patterns and their required subsections
     AMENDMENT_SECTION_TEMPLATES = {
         r"SType:": {
@@ -228,11 +234,11 @@ def _parse_sections(self):
         # Walk through tokens to find headings
         for i, token in enumerate(tokens):
             if token.type == 'heading_open':
-                # Get the heading level (h2 = level 2, h3 = level 3, etc.)
+                # Get the heading level (h1 = level 1, h2 = level 2, etc.)
                 level = int(token.tag[1])
 
-                # Only process h2 (top-level sections)
-                if level != 2:
+                # Skip h1 (document title); process h2+ for structure
+                if level < 2:
                     continue
 
                 # The next token should be 'inline' containing heading text
@@ -396,17 +402,125 @@ def _validate_amendment_structure(self):
         for section in main_sections:
             for pattern, template in self.AMENDMENT_SECTION_TEMPLATES.items():
                 if re.search(pattern, section.title):
+                    # Check if this is a Ledger Entry section for an
+                    # existing ledger entry type
+                    is_existing_ledger_entry = False
+                    if pattern == r"Ledger Entry:":
+                        is_existing_ledger_entry = (
+                            self._is_existing_ledger_entry(section.title)
+                        )
+
+                    # Check if this is a Transaction section for an
+                    # existing transaction type
+                    is_existing_transaction = False
+                    if pattern == r"Transaction:":
+                        is_existing_transaction = (
+                            self._is_existing_transaction(section.title)
+                        )
+
                     self._validate_subsections(
                         section,
                         template["required_subsections"],
-                        template["optional_subsections"]
+                        template["optional_subsections"],
+                        is_existing_ledger_entry,
+                        is_existing_transaction
                     )
 
+    def _is_existing_ledger_entry(self, section_title: str) -> bool:
+        """
+        Check if a Ledger Entry section is for an existing ledger entry.
+
+        Extracts the ledger entry name from the section title and checks
+        if the ledger entry exists on xrpl.org by fetching the URL:
+        https://xrpl.org/docs/references/protocol/ledger-data/ledger-entry-types/[lowercase]
+
+        Args:
+            section_title: The title of the section
+                          (e.g., "Ledger Entry: `AccountRoot`")
+
+        Returns:
+            True if this is an existing ledger entry type, False otherwise
+        """
+        # Extract ledger entry name from title like "Ledger Entry: `Foo`"
+        # or "2. Ledger Entry: `Foo`"
+        match = re.search(r'Ledger Entry:\s*`?([A-Za-z]+)`?', section_title)
+        if not match:
+            return False
+
+        entry_name = match.group(1)
+
+        # Convert to lowercase for URL
+        lowercase_name = entry_name.lower()
+        url = (
+            "https://xrpl.org/docs/references/protocol/"
+            f"ledger-data/ledger-entry-types/{lowercase_name}"
+        )
+
+        try:
+            # Try to fetch the URL
+            req = urllib.request.Request(url, method='HEAD')
+            with urllib.request.urlopen(req, timeout=5) as response:
+                # If we get a 200 response, the ledger entry exists
+                return response.status == 200
+        except urllib.error.HTTPError as e:
+            # 404 means the ledger entry doesn't exist
+            if e.code == 404:
+                return False
+            # Other errors (500, etc.) - assume it exists to be safe
+            return True
+        except (urllib.error.URLError, TimeoutError):
+            # Network error - assume it exists to be safe
+            return True
+
+    def _is_existing_transaction(self, section_title: str) -> bool:
+        """
+        Check if a Transaction section is for an existing transaction type.
+
+        Extracts the transaction name from the section title and checks
+        if the transaction exists on xrpl.org by fetching the URL:
+        https://xrpl.org/docs/references/protocol/transactions/types/[lowercase]
+
+        Args:
+            section_title: The title of the section (e.g., "Transaction: `Payment`")
+
+        Returns:
+            True if this is an existing transaction type, False otherwise
+        """
+        # Extract transaction name from title like "Transaction: `Payment`"
+        # or "3. Transaction: `Payment`"
+        match = re.search(r'Transaction:\s*`?([A-Za-z]+)`?', section_title)
+        if not match:
+            return False
+
+        transaction_name = match.group(1)
+
+        # Convert to lowercase for URL
+        lowercase_name = transaction_name.lower()
+        url = f"https://xrpl.org/docs/references/protocol/transactions/types/{lowercase_name}"
+
+        try:
+            # Try to fetch the URL
+            req = urllib.request.Request(url, method='HEAD')
+            with urllib.request.urlopen(req, timeout=5) as response:
+                # If we get a 200 response, the transaction exists
+                return response.status == 200
+        except urllib.error.HTTPError as e:
+            # 404 means the transaction doesn't exist
+            if e.code == 404:
+                return False
+            # Other errors (500, etc.) - assume it exists to be safe
+            return True
+        except (urllib.error.URLError, TimeoutError):
+            # Network error - assume it exists to be safe
+            return True
+
     def _validate_subsections(
         self,
         parent_section: Section,
         required_subsections: dict,
-        optional_subsections: dict
+        optional_subsections: dict,
+        is_existing_ledger_entry: bool = False,
+        is_existing_transaction: bool = False
     ):
         """Validate that a section has required subsections."""
         # Get all subsections under this parent
@@ -424,8 +538,27 @@ def _validate_subsections(
             if section.level == parent_section.level + 1:
                 subsections.append(section)
 
+        # For existing ledger entries, make certain subsections optional
+        exempted_subsections = set()
+        if is_existing_ledger_entry:
+            exempted_subsections = {
+                "Object Identifier",
+                "Ownership",
+                "Reserves",
+                "Deletion",
+                "RPC Name",
+            }
+
+        # For existing transactions, make Transaction Fee optional
+        if is_existing_transaction:
+            exempted_subsections.add("Transaction Fee")
+
         # Check for required subsections
         for sub_num, sub_title in required_subsections.items():
+            # Skip if this subsection is exempted
+            if sub_title in exempted_subsections:
+                continue
+
             # Find this subsection by title, regardless of its actual number
             found = any(
                 sub_title in s.title

templates/AMENDMENT_TEMPLATE.md

@@ -42,7 +42,9 @@ _[If your specification introduces new ledger entry objects, document each entry
 
 _[If your specification introduces new ledger entry common fields, you can have a section called `Transaction: Common Fields` before listing out any specific transactions.]_
 
-### 2.1. Object Identifier
+_[**Note for existing ledger entries:** If you are documenting changes to an existing ledger entry type that is already deployed on the XRPL mainnet (e.g., AccountRoot, RippleState, Offer, etc.), the following subsections are **optional**: Object Identifier (2.1), Ownership (2.4), Reserves (2.5), Deletion (2.6), and RPC Name (2.10). You only need to include these subsections if you are making changes to those aspects of the ledger entry.]_
+
+### 2.1. Object Identifier _(Optional if ledger entry already exists on XRPL)_
 
 **Key Space:** `0x[XXXX]` _[Specify the 16-bit hex value for the key space]_
 
@@ -77,15 +79,17 @@ _[Detailed explanation of field behavior, validation rules, etc.]_
 | ---------------- | ------------ | ---------------- |
 | `[lsfFlagName1]` | `0x[Value1]` | `[Description1]` |
 
-### 2.4. Ownership
+### 2.4. Ownership _(Optional if ledger entry already exists on XRPL)_
 
 _[Specify which AccountRoot object owns this ledger entry and how the ownership relationship is established.]_
 
 **Owner:** `[Account field name or "No owner" if this is a global object like FeeSettings]`
 
 **Directory Registration:** `[Describe how this object is registered in the owner's directory, or specify if it's a special case]`
 
-### 2.5. Reserves
+_[Note: This subsection is optional if you are documenting changes to an existing ledger entry type that is already deployed on the XRPL mainnet.]_
+
+### 2.5. Reserves _(Optional if ledger entry already exists on XRPL)_
 
 **Reserve Requirement:** `[Standard/Custom/None]`
 
@@ -95,7 +99,9 @@ _[If Custom]: This ledger entry requires `[X]` reserve units because `[reason]`.
 
 _[If None]: This ledger entry does not require additional reserves because `[reason]`._
 
-### 2.6. Deletion
+_[Note: This subsection is optional if you are documenting changes to an existing ledger entry type that is already deployed on the XRPL mainnet.]_
+
+### 2.6. Deletion _(Optional if ledger entry already exists on XRPL)_
 
 **Deletion Transactions:** `[List transaction types that can delete this object]`
 
@@ -109,6 +115,8 @@ _[If None]: This ledger entry does not require additional reserves because `[rea
 _[If Yes]: This object must be deleted before its owner account can be deleted._
 _[If No]: This object does not prevent its owner account from being deleted._
 
+_[Note: This subsection is optional if you are documenting changes to an existing ledger entry type that is already deployed on the XRPL mainnet.]_
+
 ### 2.7. Pseudo-Account _(Optional)_
 
 _[Only include this section if your ledger entry uses a pseudo-account. Otherwise, delete this subsection.]_
@@ -138,12 +146,14 @@ _[List logical statements that must always be true for this ledger entry. Use `<
 - `[Invariant 2, e.g., "IF <object>.Status == 'Active' THEN <object>.Account != NULL"]`
 - `[Additional invariants as needed]`
 
-### 2.10. RPC Name
+### 2.10. RPC Name _(Optional if ledger entry already exists on XRPL)_
 
 **RPC Type Name:** `[snake_case_name]`
 
 _[This is the name used in `account_objects` and `ledger_data` RPC calls to filter for this object type]_
 
+_[Note: This subsection is optional if you are documenting changes to an existing ledger entry type that is already deployed on the XRPL mainnet.]_
+
 ### 2.11. Example JSON
 
 ```json
@@ -165,6 +175,12 @@ _[If your specification introduces new transactions, document each transaction i
 
 _[If your specification introduces new transaction common fields, you can have a section called `Transaction: Common Fields` before listing out any specific transactions.]_
 
+> **Note for Existing Transactions:** If you are documenting changes to an existing transaction type (one that is already deployed on XRPL mainnet), the following subsection is **optional** unless you are modifying it:
+>
+> - **Transaction Fee** (3.3)
+>
+> For new transaction types, all subsections are required.
+
 > **Naming Convention:** Transaction names should follow the pattern `<LedgerEntryName><Verb>` (e.g., `ExampleSet`, `ExampleDelete`). Most specifications will need at least:
 >
 > - `[Object]Set` or `[Object]Create`: Creates or updates the object
@@ -194,7 +210,9 @@ _[Detailed explanation of field behavior, validation rules, etc.]_
 | --------------- | ------------ | ---------------- |
 | `[tfFlagName1]` | `0x[Value1]` | `[Description1]` |
 
-### 3.3. Transaction Fee
+### 3.3. Transaction Fee _(Optional if transaction already exists on XRPL)_
+
+_[This subsection is optional if you are documenting changes to an existing transaction type. Only include it if you are introducing a new transaction type or modifying the fee structure of an existing one.]_
 
 **Fee Structure:** `[Standard/Custom]`