Feat/4092/unifiedparser python support

analysis/analysers/parsers/UnifiedParser/build.gradle.kts

@@ -14,6 +14,7 @@ dependencies {
     implementation(libs.treesitter.kotlin)
     implementation(libs.treesitter.java)
     implementation(libs.treesitter.csharp)
+    implementation(libs.treesitter.python)
 
     testImplementation(libs.jsonassert)
 }

analysis/analysers/parsers/UnifiedParser/src/main/kotlin/de/maibornwolff/codecharta/analysers/parsers/unified/metriccollectors/AvailableCollectors.kt

@@ -10,5 +10,6 @@ enum class AvailableCollectors(
     JAVASCRIPT(FileExtension.JAVASCRIPT, ::JavascriptCollector),
     KOTLIN(FileExtension.KOTLIN, ::KotlinCollector),
     JAVA(FileExtension.JAVA, ::JavaCollector),
-    CSHARP(FileExtension.CSHARP, ::CSharpCollector)
+    CSHARP(FileExtension.CSHARP, ::CSharpCollector),
+    PYTHON(FileExtension.PYTHON, ::PythonCollector)
 }

analysis/analysers/parsers/UnifiedParser/src/main/kotlin/de/maibornwolff/codecharta/analysers/parsers/unified/metriccollectors/MetricCollector.kt

@@ -18,6 +18,7 @@ abstract class MetricCollector(
 ) {
     private var lastCountedCommentLine = -1
     private var lastCountedCodeLine = -1
+    private var rootNodeType: String = ""
 
     // maps a metric to its index for the more performant IntArray and the function how to calculate the metric
     private val metricInfo = mapOf(
@@ -34,6 +35,7 @@ abstract class MetricCollector(
 
     fun collectMetricsForFile(file: File): MutableNode {
         val rootNode = getRootNode(file)
+        rootNodeType = rootNode.type
 
         lastCountedCommentLine = -1
         lastCountedCodeLine = -1
@@ -57,7 +59,8 @@ abstract class MetricCollector(
     private fun getRootNode(file: File): TSNode {
         val parser = TSParser()
         parser.setLanguage(treeSitterLanguage)
-        return parser.parseString(null, file.readText()).rootNode
+        val rootNode = parser.parseString(null, file.readText()).rootNode
+        return rootNode
     }
 
     private val walkTree = DeepRecursiveFunction<Pair<TSTreeCursor, IntArray>, Unit> { (cursor, metrics) ->
@@ -67,9 +70,12 @@ abstract class MetricCollector(
         val startRow = currentNode.startPoint.row
         val endRow = currentNode.endPoint.row
 
-        for ((_, indexAndCalculateMetricForNodeFn) in metricInfo) {
-            val (index, calculateMetricForNodeFn) = indexAndCalculateMetricForNodeFn
-            metrics[index] += calculateMetricForNodeFn(currentNode, nodeType, startRow, endRow)
+        val skipRootToAvoidDoubleCountingLines = nodeType != rootNodeType
+        if (skipRootToAvoidDoubleCountingLines) {
+            for ((_, indexAndCalculateMetricForNodeFn) in metricInfo) {
+                val (index, calculateMetricForNodeFn) = indexAndCalculateMetricForNodeFn
+                metrics[index] += calculateMetricForNodeFn(currentNode, nodeType, startRow, endRow)
+            }
         }
 
         if (cursor.gotoFirstChild()) {
@@ -113,7 +119,7 @@ abstract class MetricCollector(
             rlocForNode++
         }
 
-        if (node.childCount == 0 && endRow > lastCountedCodeLine) {
+        if (endRow > lastCountedCodeLine && countWholeNodeLength(node)) {
             lastCountedCodeLine = endRow
             rlocForNode += endRow - startRow
         }
@@ -132,11 +138,20 @@ abstract class MetricCollector(
 
     private fun isNestedTypeAllowed(node: TSNode, nodeType: String, nestedTypes: Set<NestedNodeType>): Boolean {
         for (nestedType in nestedTypes) {
-            if (nestedType.baseNodeType == nodeType) {
+            if (nestedType.baseNodeType != nodeType) continue
+
+            if (nestedType.childNodePosition != null && nestedType.childNodeCount == node.childCount) {
+                val childNode = node.getChild(nestedType.childNodePosition)
+                if (nestedType.childNodeTypes.contains(childNode.type)) return true
+            } else if (nestedType.childNodeFieldName != null) {
                 val childNode = node.getChildByFieldName(nestedType.childNodeFieldName)
                 if (nestedType.childNodeTypes.contains(childNode.type)) return true
             }
         }
         return false
     }
+
+    protected open fun countWholeNodeLength(node: TSNode): Boolean {
+        return node.childCount == 0
+    }
 }

analysis/analysers/parsers/UnifiedParser/src/main/kotlin/de/maibornwolff/codecharta/analysers/parsers/unified/metriccollectors/PythonCollector.kt

@@ -0,0 +1,32 @@
+package de.maibornwolff.codecharta.analysers.parsers.unified.metriccollectors
+
+import de.maibornwolff.codecharta.analysers.parsers.unified.metricnodetypes.PythonNodeTypes
+import org.treesitter.TSNode
+import org.treesitter.TreeSitterPython
+
+class PythonCollector : MetricCollector(
+    treeSitterLanguage = TreeSitterPython(),
+    queryProvider = PythonNodeTypes()
+) {
+    override fun calculateRealLinesOfCodeForNode(node: TSNode, nodeType: String, startRow: Int, endRow: Int): Int {
+        if (shouldIgnoreNodeType(node, nodeType) || doesNodeStartWithComment(node)) return 0
+        return super.calculateRealLinesOfCodeForNode(node, nodeType, startRow, endRow)
+    }
+
+    private fun shouldIgnoreNodeType(node: TSNode, nodeType: String): Boolean {
+        return nodeType == "string_start" ||
+            nodeType == "string_content" ||
+            nodeType == "string_end" ||
+            (nodeType == "string" && node.parent.childCount == 1)
+    }
+
+    private fun doesNodeStartWithComment(node: TSNode): Boolean {
+        val childNode = node.getChild(0)
+        if (childNode.isNull) return false
+        return childNode.type == "expression_statement" && childNode.childCount == 1
+    }
+
+    override fun countWholeNodeLength(node: TSNode): Boolean {
+        return node.childCount == 0 || (node.type == "string" && node.parent.childCount != 1)
+    }
+}

analysis/analysers/parsers/UnifiedParser/src/main/kotlin/de/maibornwolff/codecharta/analysers/parsers/unified/metricnodetypes/MetricNodeTypes.kt

@@ -21,6 +21,8 @@ class TreeNodeTypes(
 
 class NestedNodeType(
     val baseNodeType: String,
-    val childNodeFieldName: String?,
+    val childNodeFieldName: String? = null,
+    val childNodeCount: Int? = null,
+    val childNodePosition: Int? = null,
     val childNodeTypes: Set<String>
 )

analysis/analysers/parsers/UnifiedParser/src/main/kotlin/de/maibornwolff/codecharta/analysers/parsers/unified/metricnodetypes/PythonNodeTypes.kt

@@ -0,0 +1,51 @@
+package de.maibornwolff.codecharta.analysers.parsers.unified.metricnodetypes
+
+class PythonNodeTypes : MetricNodeTypes {
+    override val complexityNodeTypes = TreeNodeTypes(
+        simpleNodeTypes = setOf(
+            // if
+            "if_statement",
+            "elif_clause",
+            "if_clause",
+            // loop
+            "for_statement",
+            "while_statement",
+            "for_in_clause",
+            // conditional
+            "conditional_expression",
+            "list",
+            // logical binary
+            "boolean_operator",
+            // case label
+            "case_pattern",
+            // catch block
+            "except_clause",
+            // function
+            "function_definition"
+        ),
+        nestedNodeTypes = setOf(
+            // lambda needs to be complex to not be counted double as type of first child is also lambda
+            NestedNodeType(
+                baseNodeType = "lambda",
+                childNodeCount = 4,
+                childNodePosition = 0,
+                childNodeTypes = setOf("lambda")
+            )
+        )
+    )
+
+    override val commentLineNodeTypes = TreeNodeTypes(
+        simpleNodeTypes = setOf(
+            "comment"
+        ),
+        nestedNodeTypes = setOf(
+            // in python unassigned strings are used as block comments, meaning an expression that only has string as a child
+            NestedNodeType(
+                baseNodeType = "expression_statement",
+                childNodeCount = 1,
+                childNodePosition = 0,
+                childNodeTypes = setOf("string")
+            )
+        )
+    )
+}

analysis/analysers/parsers/UnifiedParser/src/test/kotlin/de/maibornwolff/codecharta/analysers/parsers/unified/UnifiedParserTest.kt

@@ -42,16 +42,17 @@ class UnifiedParserTest {
         Arguments.of("javascript", ".js"),
         Arguments.of("java", ".java"),
         Arguments.of("kotlin", ".kt"),
-        Arguments.of("cSharp", ".cs")
+        Arguments.of("cSharp", ".cs"),
+        Arguments.of("python", ".py")
     )
 
     @ParameterizedTest
     @MethodSource("provideSupportedLanguages")
     fun `Should produce correct output for a single source file of each supported language`(language: String, fileExtension: String) {
         // given
         val pipedProject = ""
-        val inputFilePath = "${testResourceBaseFolder}${language}Sample$fileExtension"
-        val expectedResultFile = File("${testResourceBaseFolder}${language}Sample.cc.json")
+        val inputFilePath = "${testResourceBaseFolder}languageSamples/${language}Sample$fileExtension"
+        val expectedResultFile = File("${testResourceBaseFolder}languageSamples/${language}Sample.cc.json")
 
         // when
         val result = executeForOutput(pipedProject, arrayOf(inputFilePath))
@@ -135,6 +136,7 @@ class UnifiedParserTest {
             "bar/hello.kt",
             "bar/foo.kt",
             "foo.kt",
+            "foo.py",
             "whenCase.kt",
             "helloWorld.ts"
         )
@@ -166,7 +168,9 @@ class UnifiedParserTest {
 
         // then
         Assertions.assertThat(errContent.toString()).contains(
-            "2 Files with the following extensions were ignored as they are currently not supported:\n[.strange, .py]"
+            "2 Files with the following extensions were ignored as they are currently not supported:",
+            ".json",
+            ".strange"
         )
 
         // clean up