Merge pull request #47 from joaopauloschuler/copilot/test-list-directory-tree-md-files

Fix markdown section extraction to skip content inside code blocks

Author: joaopauloschuler

src/smolagents/bp_tools.py

@@ -1888,7 +1888,28 @@ def extract_function_signatures(filename: str, language: str = "python") -> str:
 
     if language.lower() in ["markdown", "md"]:
         # Extract markdown sections (lines starting with #)
-        md_sections = [line.strip() for line in content.split('\n') if line.strip().startswith('#')]
+        # Skip lines inside code blocks (fenced with ``` or ~~~)
+        md_sections = []
+        code_block_delimiter = None  # Stores the opening delimiter type
+        for line in content.split('\n'):
+            stripped = line.strip()
+            # Check for code block delimiters (``` or ~~~)
+            if code_block_delimiter is None:
+                # Not in a code block - check for opening delimiter
+                if stripped.startswith('```'):
+                    code_block_delimiter = '```'
+                    continue
+                elif stripped.startswith('~~~'):
+                    code_block_delimiter = '~~~'
+                    continue
+            else:
+                # Inside a code block - check for matching closing delimiter
+                if stripped.startswith(code_block_delimiter):
+                    code_block_delimiter = None
+                    continue
+            # Only extract headers when not inside a code block
+            if code_block_delimiter is None and stripped.startswith('#'):
+                md_sections.append(stripped)
         if not md_sections:
             return f"No sections found in '{filename}'"
         return "\n".join(md_sections)

tests/test_bp_context_tools.py

@@ -274,6 +274,38 @@ def test_function_signatures_with_non_source_files(self, tmp_path):
         assert "script.py" in result
         assert "def my_function():" in result
 
+    def test_markdown_sections_skip_code_blocks(self, tmp_path):
+        """Test that markdown section extraction skips content inside code blocks"""
+        # Create a markdown file with headers inside code blocks
+        (tmp_path / "doc.md").write_text(
+            "# Title\n"
+            "Some intro text\n"
+            "```python\n"
+            "# This is a Python comment, not a header\n"
+            "def hello():\n"
+            "    pass\n"
+            "```\n"
+            "## Section 2\n"
+            "More text\n"
+            "~~~bash\n"
+            "# Bash comment\n"
+            "echo hello\n"
+            "~~~\n"
+            "### Section 3\n"
+        )
+
+        result = list_directory_tree(str(tmp_path), max_depth=1, show_files=True, add_function_signatures=True)
+
+        # Should show the file
+        assert "doc.md" in result
+        # Should show real markdown headers
+        assert "# Title" in result
+        assert "## Section 2" in result
+        assert "### Section 3" in result
+        # Should NOT show comments inside code blocks
+        assert "# This is a Python comment" not in result
+        assert "# Bash comment" not in result
+
 
 class TestSearchInFiles:
     def test_basic_search(self, tmp_path):
@@ -629,6 +661,72 @@ def test_no_signatures_found(self, tmp_path):
 
         assert "No function/class signatures found" in result
 
+    def test_markdown_sections_basic(self, tmp_path):
+        """Test extracting markdown section headers"""
+        file_path = tmp_path / "test.md"
+        file_path.write_text("# Header 1\nSome text\n## Header 2\nMore text\n### Header 3\n")
+
+        result = extract_function_signatures(str(file_path), "markdown")
+
+        assert "# Header 1" in result
+        assert "## Header 2" in result
+        assert "### Header 3" in result
+        assert "Some text" not in result
+
+    def test_markdown_sections_skip_code_blocks(self, tmp_path):
+        """Test that markdown section extraction skips content inside code blocks"""
+        file_path = tmp_path / "test.md"
+        file_path.write_text(
+            "# Real Header\n"
+            "```python\n"
+            "# This is a comment in code block\n"
+            "def hello():\n"
+            "    pass\n"
+            "```\n"
+            "## Another Real Header\n"
+            "~~~bash\n"
+            "# Bash comment\n"
+            "~~~\n"
+            "### Third Header\n"
+        )
+
+        result = extract_function_signatures(str(file_path), "markdown")
+
+        assert "# Real Header" in result
+        assert "## Another Real Header" in result
+        assert "### Third Header" in result
+        # Comments inside code blocks should NOT be extracted
+        assert "# This is a comment in code block" not in result
+        assert "# Bash comment" not in result
+
+    def test_markdown_no_sections(self, tmp_path):
+        """Test markdown file with no section headers"""
+        file_path = tmp_path / "test.md"
+        file_path.write_text("Just plain text\nNo headers here\n")
+
+        result = extract_function_signatures(str(file_path), "markdown")
+
+        assert "No sections found" in result
+
+    def test_markdown_mixed_delimiters(self, tmp_path):
+        """Test that mixed code block delimiters are handled correctly"""
+        file_path = tmp_path / "test.md"
+        # In proper markdown, ``` must be closed with ``` and ~~~ with ~~~
+        # So a ~~~ should NOT close a block opened with ```
+        file_path.write_text(
+            "# Header 1\n"
+            "```python\n"
+            "# comment in code\n"
+            "~~~\n"  # This should NOT close the block
+            "## Should be skipped (still in code block)\n"
+        )
+
+        result = extract_function_signatures(str(file_path), "markdown")
+
+        # Only the first header should be extracted since the code block is never properly closed
+        assert "# Header 1" in result
+        assert "## Should be skipped" not in result
+
     def test_nonexistent_file(self):
         """Test nonexistent file"""
         result = extract_function_signatures("/nonexistent/file.py", "python")