feat: add backtick standardization to clean up markdown-code-runner tags in output files and protect them when doing an in-place replacement.

Needed because, systems like mkdocs, pandoc are unable to parse fenced blocks if they have extra arguments other than the language to be highlighted.

Author: sanyamsmulay

.gitignore

@@ -0,0 +1,39 @@
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# Virtual Environment
+venv/
+venv-*
+env/
+ENV/
+
+# IDE
+.idea/
+.vscode/
+*.swp
+*.swo
+
+# Misc
+.DS_Store
+.env
+.coverage
+htmlcov/

markdown_code_runner.py

@@ -34,7 +34,7 @@
 ```bash markdown-code-runner
 echo "Hello, world!"
 ```
-Which will similarly print the output of the code block between next to the output markers.
+Which will similarly print the output of the code block between the next output markers.
 
 """
 
@@ -45,6 +45,7 @@
 import io
 import os
 import re
+import sys
 import subprocess
 from dataclasses import dataclass, field
 from importlib.metadata import PackageNotFoundError, version
@@ -78,6 +79,12 @@ def md_comment(text: str) -> str:
     "code:backticks:end": "```",
 }
 
+# List of all start markers for easier checking
+START_MARKERS = [
+    marker for marker in MARKERS 
+    if marker.endswith(":start")
+]
+
 
 def markers_to_patterns() -> dict[str, re.Pattern]:
     """Convert the markers to regular expressions."""
@@ -165,21 +172,30 @@ def _extract_backtick_options(line: str) -> dict[str, str]:
     """Extract extra information from a line."""
     if "```" not in line:
         return {}
+    
+    # First try to match with markdown-code-runner
     language_pattern = r"```(?P<language>\w+) markdown-code-runner"
-    extra_pattern = r"(?P<key>\w+)=(?P<value>\S+)"
-
     language_match = re.search(language_pattern, line)
-    assert language_match is not None
+    
+    # If no match, try to match just the language
+    if language_match is None:
+        language_pattern = r"```(?P<language>\w+)"
+        language_match = re.search(language_pattern, line)
+        if language_match is None:
+            return {}
+    
     language = language_match.group("language")
     result = {"language": language}
-
-    extra_str = line[language_match.end() :]
-    extra_matches = re.finditer(extra_pattern, extra_str)
-
-    for match in extra_matches:
-        key, value = match.group("key"), match.group("value")
-        result[key] = value
-
+    
+    # Only look for extra options if markdown-code-runner is present
+    if "markdown-code-runner" in line:
+        extra_pattern = r"(?P<key>\w+)=(?P<value>\S+)"
+        extra_str = line[language_match.end() :]
+        extra_matches = re.finditer(extra_pattern, extra_str)
+        for match in extra_matches:
+            key, value = match.group("key"), match.group("value")
+            result[key] = value
+    
     return result
 
 
@@ -203,6 +219,7 @@ class ProcessingState:
     output: list[str] | None = None
     new_lines: list[str] = field(default_factory=list)
     backtick_options: dict[str, Any] = field(default_factory=dict)
+    backtick_standardize: bool = True
 
     def process_line(self, line: str, *, verbose: bool = False) -> None:
         """Process a line of the Markdown file."""
@@ -219,19 +236,46 @@ def process_line(self, line: str, *, verbose: bool = False) -> None:
         elif self.section == "output":
             self.original_output.append(line)
         else:
-            self._process_start_markers(line)
+            processed_line = self._process_start_markers(line, verbose=verbose)
+            if processed_line: 
+                line = processed_line
 
         if self.section != "output":
             self.new_lines.append(line)
 
-    def _process_start_markers(self, line: str) -> None:
-        for marker in MARKERS:
-            if marker.endswith(":start") and is_marker(line, marker):
+    def _process_start_markers(self, line: str, verbose: bool = False) -> None:
+        for marker in START_MARKERS:
+            if is_marker(line, marker):
                 # reset output in case previous output wasn't displayed
                 self.output = None
                 self.backtick_options = _extract_backtick_options(line)
                 self.section, _ = marker.rsplit(":", 1)  # type: ignore[assignment]
-                return
+                processed_line = line
+                if marker == "code:backticks:start":
+                    if verbose:
+                        print(f"Found marker {marker} in line {line}")
+                    processed_line = self._process_backticks_start(line)
+                return processed_line
+            
+    def _process_backticks_start(self, line: str) -> str:
+        """Process backticks start marker and standardize if needed.
+        
+        Args:
+            line: The line containing backticks start marker
+            
+        Returns:
+            Processed line with markdown-code-runner removed if standardization is enabled
+        """
+        language_match = re.search(r"```(?P<language>\w+)", line)
+        if not (language_match and self.backtick_standardize):
+            return line
+            
+        if "markdown-code-runner" not in line:
+            return line
+            
+        # Remove markdown-code-runner and any text after it from the line
+        processed_line = re.sub(r'\smarkdown-code-runner.*(?=```|$)', '', line)
+        return processed_line
 
     def _process_output_start(self, line: str) -> None:
         self.section = "output"
@@ -292,7 +336,7 @@ def _process_backtick_code(self, line: str, *, verbose: bool) -> None:
         self._process_code(line, "code:backticks:end", language, verbose=verbose)
 
 
-def process_markdown(content: list[str], *, verbose: bool = False) -> list[str]:
+def process_markdown(content: list[str], *, verbose: bool = False, backtick_standardize: bool = True) -> list[str]:
     """Executes code blocks in a list of Markdown-formatted strings and returns the modified list.
 
     Parameters
@@ -301,6 +345,8 @@ def process_markdown(content: list[str], *, verbose: bool = False) -> list[str]:
         A list of Markdown-formatted strings.
     verbose
         If True, print every line that is processed.
+    backtick_standardize
+        If True, clean up markdown-code-runner string from backtick code blocks.
 
     Returns
     -------
@@ -309,14 +355,13 @@ def process_markdown(content: list[str], *, verbose: bool = False) -> list[str]:
 
     """
     assert isinstance(content, list), "Input must be a list"
-    state = ProcessingState()
+    state = ProcessingState(backtick_standardize=backtick_standardize)
 
     for i, line in enumerate(content):
         if verbose:
             nr = _bold(f"line {i:4d}")
             print(f"{nr}: {line}")
-        state.process_line(line, verbose=verbose)
-
+        line = state.process_line(line, verbose=verbose)
     return state.new_lines
 
 
@@ -325,15 +370,28 @@ def update_markdown_file(
     output_filepath: Path | str | None = None,
     *,
     verbose: bool = False,
+    backtick_standardize: bool = True,
 ) -> None:
-    """Rewrite a Markdown file by executing and updating code blocks."""
+    """Rewrite a Markdown file by executing and updating code blocks.
+    
+    Parameters
+    ----------
+    input_filepath : Path | str
+        Path to the input Markdown file.
+    output_filepath : Path | str | None
+        Path to the output Markdown file. If None, overwrites input file.
+    verbose : bool
+        If True, print every line that is processed.
+    backtick_standardize : bool
+        If True, clean up markdown-code-runner string from backtick code blocks.
+    """
     if isinstance(input_filepath, str):  # pragma: no cover
         input_filepath = Path(input_filepath)
     with input_filepath.open() as f:
         original_lines = [line.rstrip("\n") for line in f.readlines()]
     if verbose:
         print(f"Processing input file: {input_filepath}")
-    new_lines = process_markdown(original_lines, verbose=verbose)
+    new_lines = process_markdown(original_lines, verbose=verbose, backtick_standardize=backtick_standardize)
     updated_content = "\n".join(new_lines).rstrip() + "\n"
     if verbose:
         print(f"Writing output to: {output_filepath}")
@@ -375,12 +433,34 @@ def main() -> None:
         action="version",
         version=f"%(prog)s {__version__}",
     )
+    parser.add_argument(
+        "--backtick-standardize",
+        action="store_true",
+        help="Clean up markdown-code-runner string from backtick code blocks (default: True when output file is specified)",
+        default=None,
+    )
+    parser.add_argument(
+        "--force-overwrite",
+        action="store_true",
+        help="Required when using backtick-standardize option with input file overwrite",
+        default=False,
+    )
 
     args = parser.parse_args()
 
     input_filepath = Path(args.input)
     output_filepath = Path(args.output) if args.output is not None else input_filepath
-    update_markdown_file(input_filepath, output_filepath, verbose=args.verbose)
+    
+    # Determine backtick standardization
+    if args.output is None:  # Overwriting input file
+        if args.backtick_standardize and not args.force_overwrite:
+            print("Error: This will overwrite your file. Please use the --force-overwrite option in conjunction to set backtick-standardize option to true", file=sys.stderr)
+            sys.exit(1)
+        backtick_standardize = args.backtick_standardize and args.force_overwrite
+    else:  # Writing to different output file
+        backtick_standardize = args.backtick_standardize if args.backtick_standardize is not None else True
+    
+    update_markdown_file(input_filepath, output_filepath, verbose=args.verbose, backtick_standardize=backtick_standardize)
 
 
 if __name__ == "__main__":

tests/test_backtick_stdz.md

@@ -0,0 +1,48 @@
+# Backtick Standardization Test
+
+This file tests various backtick code block scenarios.
+
+## Basic Code Block
+A simple Python code block with markdown-code-runner:  
+Currently no options are supported for backtick code blocks. May be used in the future.
+
+```python markdown-code-runner filename=test1.py
+print("Basic test")
+```
+
+## Code Block with Multiple Options
+Testing multiple options in the backtick header:
+
+```javascript markdown-code-runner filename=test2.js debug=true skip=false
+console.log("Multiple options test")
+```
+
+## Language-only Block
+This block should remain unchanged during standardization:
+
+```rust
+fn main() {
+    println!("No markdown-code-runner");
+}
+```
+
+## Complex Options Block
+Testing complex options and spacing:
+
+```python   markdown-code-runner    filename=test3.py   debug=true   skip=false   
+print("Testing spaces in options")
+```
+
+## Empty Language Block
+Testing block with no language:
+
+```markdown-code-runner filename=test4.txt
+Just some plain text
+```
+
+## Mixed Content Block
+Testing with additional content after options:
+
+```python markdown-code-runner filename=test5.py some random text here
+print("Mixed content test")
+```