fix: 🐛 Fix nocl exclusions

Author: robvanderleek

codelimit/common/Scanner.py

@@ -9,8 +9,8 @@
 from pygments.lexer import Lexer
 from pygments.lexers import get_lexer_for_filename
 from pygments.util import ClassNotFound
-from rich.live import Live
 from rich import print
+from rich.live import Live
 
 from codelimit.common.Codebase import Codebase
 from codelimit.common.Configuration import Configuration
@@ -153,7 +153,7 @@ def scan_file(tokens: list[Token], language: Language) -> list[Measurement]:
                 last_token.location.column + len(last_token.value),
             )
             measurements.append(
-                Measurement(scope.header.name, start_location, end_location, length)
+                Measurement(scope.header.name(), start_location, end_location, length)
             )
     return measurements
 

codelimit/common/scope/Header.py

@@ -6,9 +6,12 @@
 
 @dataclass
 class Header:
-    name: str
+    name_token: Token
     token_range: TokenRange
 
+    def name(self) -> str:
+        return self.name_token.value
+
 
 def sort_headers(headers: list[Header], tokens: list[Token], reverse=False) -> list[Header]:
     return sorted(

codelimit/common/scope/scope_utils.py

@@ -18,7 +18,7 @@ def build_scopes(tokens: list[Token], language: Language) -> list[Scope]:
     headers = language.extract_headers(code_tokens)
     blocks = language.extract_blocks(code_tokens, headers)
     scopes = _build_scopes_from_headers_and_blocks(headers, blocks, code_tokens)
-    filtered_scopes = _filter_nocl_scopes(scopes, code_tokens, nocl_comment_tokens)
+    filtered_scopes = _filter_nocl_scopes(scopes, nocl_comment_tokens)
     if language.allow_nested_functions:
         return fold_scopes(filtered_scopes)
     else:
@@ -105,24 +105,10 @@ def _get_nearest_block(
 
 
 def _filter_nocl_scopes(
-        scopes: list[Scope], tokens: list[Token], nocl_comment_tokens: list[Token]
+        scopes: list[Scope], nocl_comment_tokens: list[Token]
 ) -> list[Scope]:
     nocl_comment_lines = [t.location.line for t in nocl_comment_tokens]
-
-    def get_scope_header_lines(scope: Scope) -> set[int]:
-        header_token_range = scope.header.token_range
-        header_tokens = tokens[header_token_range.start:header_token_range.end]
-        result = set([t.location.line for t in header_tokens])
-        first_line = header_tokens[0].location.line
-        if first_line > 0:
-            result.add(first_line - 1)
-        return result
-
-    return [
-        s
-        for s in scopes
-        if len(get_scope_header_lines(s).intersection(nocl_comment_lines)) == 0
-    ]
+    return [s for s in scopes if s.header.name_token.location.line not in nocl_comment_lines]
 
 
 def has_name_prefix(tokens: list[Token], index: int) -> bool:
@@ -144,7 +130,7 @@ def get_headers(
     for pattern in patterns:
         name_token = next(t for t in pattern.tokens if t.is_name())
         if name_token:
-            result.append(Header(name_token.value, TokenRange(pattern.start, pattern.end)))
+            result.append(Header(name_token, TokenRange(pattern.start, pattern.end)))
     return result
 
 

codelimit/common/source_utils.py

@@ -65,7 +65,11 @@ def filter_nocl_comment_tokens(tokens: list[Token]):
     def predicate(token: Token):
         if token.is_comment():
             value = token.value.lower()
-            return value.startswith("#nocl") or value.startswith("# nocl")
+            if value.startswith("#") or value.startswith(";"):
+                value = value[1:].strip()
+            elif value.startswith("//") or value.startswith("/*"):
+                value = value[2:].strip()
+            return value.startswith("nocl")
         else:
             return False
 

tests/common/test_source_utils.py

@@ -1,4 +1,5 @@
 from pygments.lexers import PythonLexer
+from pygments.lexers import CppLexer
 
 from codelimit.common.Location import Location
 from codelimit.common.lexer_utils import lex
@@ -116,3 +117,16 @@ def test_filter_nocl_comments():
 
     assert len(result) == 1
     assert result[0].location.line == 1
+
+    code = ""
+    code += "// nocl\n"
+    code += "void\n"
+    code += "foo(Bar bar) {\n"
+    code += "  bar.foo();\n"
+    code += "}\n"
+    tokens = lex(CppLexer(), code, False)
+
+    result = filter_nocl_comment_tokens(tokens)
+
+    assert len(result) == 1
+    assert result[0].location.line == 1

tests/conftest.py

@@ -17,8 +17,8 @@ def assert_units(code: str, language: Language, units: dict[str, int]):
     scopes = unfold_scopes(scopes)
     assert len(scopes) == len(units)
     for idx, scope in enumerate(scopes):
-        assert scope.header.name in units
-        assert count_lines(scope, code_tokens) == units[scope.header.name]
+        assert scope.header.name() in units
+        assert count_lines(scope, code_tokens) == units[scope.header.name()]
 
 
 def print_units(code: str, language: Language):

tests/languages/test_Cpp.py

@@ -72,3 +72,21 @@ class Main {
     """
 
     assert_units(code, Languages.Cpp, {"sayHello": 3})
+
+
+def test_skip_function_with_nocl_comment():
+    code = """
+    void foo(Bar bar) {
+        bar.foo();
+    }
+    """
+
+    assert_units(code, Languages.Cpp, {"foo": 3})
+
+    code = """
+    void foo(Bar bar) { // nocl
+        bar.foo();
+    }
+    """
+
+    assert_units(code, Languages.Cpp, {})

tests/languages/test_Python.py

@@ -147,37 +147,19 @@ def foo(
 
 def test_skip_function_with_nocl_comment_in_header():
     code = """
-    def bar(
-        bar: Bar
-    ) -> JSONResponse: # nocl
-        bar = foo
-    
-    def foo(
-        foo: Foo
-    ) -> None:
-        foo = bar
-        bar = foo
-    """
-
-    assert_units(code, Languages.Python, {"foo": 5})
-
-
-def test_skip_function_with_nocl_comment_before_header():
-    code = """
-    def bar(
+    def bar( # NOCL
         bar: Bar
     ) -> JSONResponse:
         bar = foo
     
-    # NOCL
     def foo(
         foo: Foo
     ) -> None:
         foo = bar
         bar = foo
     """
 
-    assert_units(code, Languages.Python, {"bar": 4})
+    assert_units(code, Languages.Python, {"foo": 5})
 
 
 def test_function_with_type_hints():