Merge pull request github#10759 from igfoo/igfoo/numlines2

igfoo · web-flow · commit f0eabd467508 · 2022-10-14T15:39:40.000+01:00
kotlin: Populate numlines
diff --git a/java/kotlin-extractor/src/main/kotlin/ExternalDeclExtractor.kt b/java/kotlin-extractor/src/main/kotlin/ExternalDeclExtractor.kt
@@ -84,7 +84,7 @@ class ExternalDeclExtractor(val logger: FileLogger, val invocationTrapFile: Stri
                                     // file information if needed:
                                     val ftw = tw.makeFileTrapWriter(binaryPath, irDecl is IrClass)
 
-                                    val fileExtractor = KotlinFileExtractor(logger, ftw, binaryPath, manager, this, primitiveTypeMapping, pluginContext, KotlinFileExtractor.DeclarationStack(), globalExtensionState)
+                                    val fileExtractor = KotlinFileExtractor(logger, ftw, null, binaryPath, manager, this, primitiveTypeMapping, pluginContext, KotlinFileExtractor.DeclarationStack(), globalExtensionState)
 
                                     if (irDecl is IrClass) {
                                         // Populate a location and compilation-unit package for the file. This is similar to
diff --git a/java/kotlin-extractor/src/main/kotlin/KotlinExtractorExtension.kt b/java/kotlin-extractor/src/main/kotlin/KotlinExtractorExtension.kt
@@ -324,7 +324,8 @@ private fun doFile(
                 // file information
                 val sftw = tw.makeSourceFileTrapWriter(srcFile, true)
                 val externalDeclExtractor = ExternalDeclExtractor(logger, invocationTrapFile, srcFilePath, primitiveTypeMapping, pluginContext, globalExtensionState, fileTrapWriter)
-                val fileExtractor = KotlinFileExtractor(logger, sftw, srcFilePath, null, externalDeclExtractor, primitiveTypeMapping, pluginContext, KotlinFileExtractor.DeclarationStack(), globalExtensionState)
+                val linesOfCode = LinesOfCode(logger, sftw, srcFile)
+                val fileExtractor = KotlinFileExtractor(logger, sftw, linesOfCode, srcFilePath, null, externalDeclExtractor, primitiveTypeMapping, pluginContext, KotlinFileExtractor.DeclarationStack(), globalExtensionState)
 
                 fileExtractor.extractFileContents(srcFile, sftw.fileId)
                 externalDeclExtractor.extractExternalClasses()
diff --git a/java/kotlin-extractor/src/main/kotlin/KotlinFileExtractor.kt b/java/kotlin-extractor/src/main/kotlin/KotlinFileExtractor.kt
@@ -36,6 +36,7 @@ import kotlin.collections.ArrayList
 open class KotlinFileExtractor(
     override val logger: FileLogger,
     override val tw: FileTrapWriter,
+    val linesOfCode: LinesOfCode?,
     val filePath: String,
     dependencyCollector: OdasaOutput.TrapFileManager?,
     externalClassExtractor: ExternalDeclExtractor,
@@ -91,6 +92,8 @@ open class KotlinFileExtractor(
             if (!declarationStack.isEmpty()) {
                 logger.errorElement("Declaration stack is not empty after processing the file", file)
             }
+
+            linesOfCode?.linesOfCodeInFile(id)
         }
     }
 
@@ -472,6 +475,8 @@ open class KotlinFileExtractor(
                 extractClassModifiers(c, id)
                 extractClassSupertypes(c, id, inReceiverContext = true) // inReceiverContext = true is specified to force extraction of member prototypes of base types
 
+                linesOfCode?.linesOfCodeInDeclaration(c, id)
+
                 return id
             }
         }
@@ -1148,6 +1153,8 @@ open class KotlinFileExtractor(
                     addModifiers(id, "suspend")
                 }
 
+                linesOfCode?.linesOfCodeInDeclaration(f, id)
+
                 return id
             }
         }
diff --git a/java/kotlin-extractor/src/main/kotlin/KotlinUsesExtractor.kt b/java/kotlin-extractor/src/main/kotlin/KotlinUsesExtractor.kt
@@ -138,13 +138,13 @@ open class KotlinUsesExtractor(
             val newTrapWriter = tw.makeFileTrapWriter(filePath, true)
             val newLoggerTrapWriter = logger.tw.makeFileTrapWriter(filePath, false)
             val newLogger = FileLogger(logger.loggerBase, newLoggerTrapWriter)
-            return KotlinFileExtractor(newLogger, newTrapWriter, filePath, dependencyCollector, externalClassExtractor, primitiveTypeMapping, pluginContext, newDeclarationStack, globalExtensionState)
+            return KotlinFileExtractor(newLogger, newTrapWriter, null, filePath, dependencyCollector, externalClassExtractor, primitiveTypeMapping, pluginContext, newDeclarationStack, globalExtensionState)
         }
 
         val newTrapWriter = tw.makeSourceFileTrapWriter(clsFile, true)
         val newLoggerTrapWriter = logger.tw.makeSourceFileTrapWriter(clsFile, false)
         val newLogger = FileLogger(logger.loggerBase, newLoggerTrapWriter)
-        return KotlinFileExtractor(newLogger, newTrapWriter, clsFile.path, dependencyCollector, externalClassExtractor, primitiveTypeMapping, pluginContext, newDeclarationStack, globalExtensionState)
+        return KotlinFileExtractor(newLogger, newTrapWriter, null, clsFile.path, dependencyCollector, externalClassExtractor, primitiveTypeMapping, pluginContext, newDeclarationStack, globalExtensionState)
     }
 
     // The Kotlin compiler internal representation of Outer<T>.Inner<S>.InnerInner<R> is InnerInner<R, S, T>. This function returns just `R`.
diff --git a/java/kotlin-extractor/src/main/kotlin/LinesOfCode.kt b/java/kotlin-extractor/src/main/kotlin/LinesOfCode.kt
@@ -0,0 +1,127 @@
+package com.github.codeql
+
+import com.github.codeql.utils.versions.Psi2Ir
+import com.intellij.psi.PsiComment
+import com.intellij.psi.PsiElement
+import com.intellij.psi.PsiWhiteSpace
+import org.jetbrains.kotlin.ir.IrElement
+import org.jetbrains.kotlin.ir.declarations.*
+import org.jetbrains.kotlin.kdoc.psi.api.KDocElement
+import org.jetbrains.kotlin.psi.KtCodeFragment
+import org.jetbrains.kotlin.psi.KtVisitor
+
+class LinesOfCode(
+    val logger: FileLogger,
+    val tw: FileTrapWriter,
+    val file: IrFile
+) {
+    val psi2Ir = Psi2Ir(logger)
+
+    fun linesOfCodeInFile(id: Label<DbFile>) {
+        val ktFile = psi2Ir.getKtFile(file)
+        if (ktFile == null) {
+            return
+        }
+        linesOfCodeInPsi(id, ktFile, file)
+    }
+
+    fun linesOfCodeInDeclaration(d: IrDeclaration, id: Label<out DbSourceline>) {
+        val p = psi2Ir.findPsiElement(d, file)
+        if (p == null) {
+            return
+        }
+        linesOfCodeInPsi(id, p, d)
+    }
+
+    private fun linesOfCodeInPsi(id: Label<out DbSourceline>, root: PsiElement, e: IrElement) {
+        val document = root.getContainingFile().getViewProvider().getDocument()
+        if (document == null) {
+            logger.errorElement("Cannot find document for PSI", e)
+            tw.writeNumlines(id, 0, 0, 0)
+            return
+        }
+
+        val rootRange = root.getTextRange()
+        val rootFirstLine = document.getLineNumber(rootRange.getStartOffset())
+        val rootLastLine = document.getLineNumber(rootRange.getEndOffset())
+        if (rootLastLine < rootFirstLine) {
+            logger.errorElement("PSI ends before it starts", e)
+            tw.writeNumlines(id, 0, 0, 0)
+            return
+        }
+        val numLines = 1 + rootLastLine - rootFirstLine
+        val lineContents = Array(numLines) { LineContent() }
+
+        val visitor =
+            object : KtVisitor<Unit, Unit>() {
+                override fun visitElement(element: PsiElement) {
+                    val isComment = element is PsiComment
+                    // Comments may include nodes that aren't PsiComments,
+                    // so we don't want to visit them or we'll think they
+                    // are code.
+                    if (!isComment) {
+                        element.acceptChildren(this)
+                    }
+
+                    if (element is PsiWhiteSpace) {
+                        return
+                    }
+                    // Leaf nodes are assumed to be tokens, and
+                    // therefore we count any lines that they are on.
+                    // For comments, we actually need to look at the
+                    // outermost node, as the leaves of KDocs don't
+                    // necessarily cover all lines.
+                    if (isComment || element.getChildren().size == 0) {
+                        val range = element.getTextRange()
+                        val startOffset = range.getStartOffset()
+                        val endOffset = range.getEndOffset()
+                        // The PSI doesn't seem to have anything like
+                        // the IR's UNDEFINED_OFFSET and SYNTHETIC_OFFSET,
+                        // but < 0 still seem to represent bad/unknown
+                        // locations.
+                        if (startOffset < 0 || endOffset < 0) {
+                            logger.errorElement("PSI has negative offset", e)
+                            return
+                        }
+                        if (startOffset > endOffset) {
+                            return
+                        }
+                        // We might get e.g. an import list for a file
+                        // with no imports, which claims to have start
+                        // and end offsets of 0. Anything of 0 width
+                        // we therefore just skip.
+                        if (startOffset == endOffset) {
+                            return
+                        }
+                        val firstLine = document.getLineNumber(startOffset)
+                        val lastLine = document.getLineNumber(endOffset)
+                        if (firstLine < rootFirstLine) {
+                            logger.errorElement("PSI element starts before root", e)
+                            return
+                        } else if (lastLine > rootLastLine) {
+                            logger.errorElement("PSI element ends after root", e)
+                            return
+                        }
+                        for (line in firstLine..lastLine) {
+                            val lineContent = lineContents[line - rootFirstLine]
+                            if (isComment) {
+                                lineContent.containsComment = true
+                            } else {
+                                lineContent.containsCode = true
+                            }
+                        }
+                    }
+                }
+            }
+        root.accept(visitor)
+        val total = lineContents.size
+        val code = lineContents.count { it.containsCode }
+        val comment = lineContents.count { it.containsComment }
+        tw.writeNumlines(id, total, code, comment)
+    }
+
+    private class LineContent {
+        var containsComment = false
+        var containsCode = false
+    }
+}
diff --git a/java/ql/test/kotlin/library-tests/inherited-callee/Test.kt b/java/ql/test/kotlin/library-tests/inherited-callee/Test.kt
@@ -4,13 +4,13 @@ open class TestKt {
 
 }
 
-interface ParentIf {
+interface ParentIfK {
 
   fun inheritedInterfaceMethodK()
 
 }
 
-interface ChildIf : ParentIf {
+interface ChildIfK : ParentIfK {
 
 
 }
@@ -24,7 +24,7 @@ class ChildKt : TestKt() {
     c.equals(c)
     c.hashCode()
     c.inheritMe()
-    val c2: ParentIf? = null
+    val c2: ParentIfK? = null
     c2?.inheritedInterfaceMethodK()
 
   }
diff --git a/java/ql/test/kotlin/library-tests/inherited-callee/test.expected b/java/ql/test/kotlin/library-tests/inherited-callee/test.expected
@@ -7,4 +7,4 @@
 | Test.kt:24:7:24:15 | equals(...) | equals | Object |
 | Test.kt:25:7:25:16 | hashCode(...) | hashCode | Object |
 | Test.kt:26:7:26:17 | inheritMe(...) | inheritMe | TestKt |
-| Test.kt:28:9:28:35 | inheritedInterfaceMethodK(...) | inheritedInterfaceMethodK | ParentIf |
+| Test.kt:28:9:28:35 | inheritedInterfaceMethodK(...) | inheritedInterfaceMethodK | ParentIfK |
diff --git a/java/ql/test/kotlin/library-tests/numlines/callable.expected b/java/ql/test/kotlin/library-tests/numlines/callable.expected
@@ -0,0 +1,6 @@
+| test.kt:2:1:4:1 | foo | 3 | 3 | 0 |
+| test.kt:8:1:8:9 | getX | 1 | 1 | 0 |
+| test.kt:18:1:18:17 | getY | 5 | 1 | 4 |
+| test.kt:20:1:26:1 | Foo | 7 | 6 | 1 |
+| test.kt:21:5:24:5 | bar | 4 | 3 | 1 |
+| test.kt:25:5:25:21 | getSomeField | 1 | 1 | 0 |
diff --git a/java/ql/test/kotlin/library-tests/numlines/callable.ql b/java/ql/test/kotlin/library-tests/numlines/callable.ql
@@ -0,0 +1,4 @@
+import java
+
+from Callable c
+select c, c.getTotalNumberOfLines(), c.getNumberOfLinesOfCode(), c.getNumberOfCommentLines()
diff --git a/java/ql/test/kotlin/library-tests/numlines/classes.expected b/java/ql/test/kotlin/library-tests/numlines/classes.expected
@@ -0,0 +1 @@
+| test.kt:20:1:26:1 | Foo | 7 | 6 | 1 |
diff --git a/java/ql/test/kotlin/library-tests/numlines/classes.ql b/java/ql/test/kotlin/library-tests/numlines/classes.ql
@@ -0,0 +1,4 @@
+import java
+
+from Class c
+select c, c.getTotalNumberOfLines(), c.getNumberOfLinesOfCode(), c.getNumberOfCommentLines()
diff --git a/java/ql/test/kotlin/library-tests/numlines/files.expected b/java/ql/test/kotlin/library-tests/numlines/files.expected
@@ -0,0 +1 @@
+| test.kt:0:0:0:0 | test | 28 | 11 | 9 |
diff --git a/java/ql/test/kotlin/library-tests/numlines/files.ql b/java/ql/test/kotlin/library-tests/numlines/files.ql
@@ -0,0 +1,4 @@
+import java
+
+from File f
+select f, f.getTotalNumberOfLines(), f.getNumberOfLinesOfCode(), f.getNumberOfCommentLines()
diff --git a/java/ql/test/kotlin/library-tests/numlines/test.kt b/java/ql/test/kotlin/library-tests/numlines/test.kt
@@ -0,0 +1,27 @@
+
+fun   foo  (  x  :   Int  )  {
+    val   y  =   (  x   +   1  )
+}
+
+// test
+
+val x = 4
+
+/*
+test
+*/
+
+/**
+test
+*/
+
+val y = 5 // test
+
+class Foo {
+    fun bar() {
+        // comment
+        return
+    }
+    val someField = 3
+}
+

Original file line number	Diff line number	Diff line change
`@@ -36,6 +36,7 @@ import kotlin.collections.ArrayList`
`36`	`36`	`open class KotlinFileExtractor(`
`37`	`37`	`override val logger: FileLogger,`
`38`	`38`	`override val tw: FileTrapWriter,`
	`39`	`+ val linesOfCode: LinesOfCode?,`
`39`	`40`	`val filePath: String,`
`40`	`41`	`dependencyCollector: OdasaOutput.TrapFileManager?,`
`41`	`42`	`externalClassExtractor: ExternalDeclExtractor,`
`@@ -91,6 +92,8 @@ open class KotlinFileExtractor(`
`91`	`92`	`if (!declarationStack.isEmpty()) {`
`92`	`93`	`logger.errorElement("Declaration stack is not empty after processing the file", file)`
`93`	`94`	`}`
	`95`	`+`
	`96`	`+ linesOfCode?.linesOfCodeInFile(id)`
`94`	`97`	`}`
`95`	`98`	`}`
`96`	`99`
`@@ -472,6 +475,8 @@ open class KotlinFileExtractor(`
`472`	`475`	`extractClassModifiers(c, id)`
`473`	`476`	`extractClassSupertypes(c, id, inReceiverContext = true) // inReceiverContext = true is specified to force extraction of member prototypes of base types`
`474`	`477`
	`478`	`+ linesOfCode?.linesOfCodeInDeclaration(c, id)`
	`479`	`+`
`475`	`480`	`return id`
`476`	`481`	`}`
`477`	`482`	`}`
`@@ -1148,6 +1153,8 @@ open class KotlinFileExtractor(`
`1148`	`1153`	`addModifiers(id, "suspend")`
`1149`	`1154`	`}`
`1150`	`1155`
	`1156`	`+ linesOfCode?.linesOfCodeInDeclaration(f, id)`
	`1157`	`+`
`1151`	`1158`	`return id`
`1152`	`1159`	`}`
`1153`	`1160`	`}`
Original file line number	Diff line number	Diff line change
`@@ -138,13 +138,13 @@ open class KotlinUsesExtractor(`
`138`	`138`	`val newTrapWriter = tw.makeFileTrapWriter(filePath, true)`
`139`	`139`	`val newLoggerTrapWriter = logger.tw.makeFileTrapWriter(filePath, false)`
`140`	`140`	`val newLogger = FileLogger(logger.loggerBase, newLoggerTrapWriter)`
`141`		`- return KotlinFileExtractor(newLogger, newTrapWriter, filePath, dependencyCollector, externalClassExtractor, primitiveTypeMapping, pluginContext, newDeclarationStack, globalExtensionState)`
	`141`	`+ return KotlinFileExtractor(newLogger, newTrapWriter, null, filePath, dependencyCollector, externalClassExtractor, primitiveTypeMapping, pluginContext, newDeclarationStack, globalExtensionState)`
`142`	`142`	`}`
`143`	`143`
`144`	`144`	`val newTrapWriter = tw.makeSourceFileTrapWriter(clsFile, true)`
`145`	`145`	`val newLoggerTrapWriter = logger.tw.makeSourceFileTrapWriter(clsFile, false)`
`146`	`146`	`val newLogger = FileLogger(logger.loggerBase, newLoggerTrapWriter)`
`147`		`- return KotlinFileExtractor(newLogger, newTrapWriter, clsFile.path, dependencyCollector, externalClassExtractor, primitiveTypeMapping, pluginContext, newDeclarationStack, globalExtensionState)`
	`147`	`+ return KotlinFileExtractor(newLogger, newTrapWriter, null, clsFile.path, dependencyCollector, externalClassExtractor, primitiveTypeMapping, pluginContext, newDeclarationStack, globalExtensionState)`
`148`	`148`	`}`
`149`	`149`
`150`	`150`	// The Kotlin compiler internal representation of Outer<T>.Inner<S>.InnerInner<R> is InnerInner<R, S, T>. This function returns just `R`.
Original file line number	Diff line number	Diff line change
`@@ -4,13 +4,13 @@ open class TestKt {`
`4`	`4`
`5`	`5`	`}`
`6`	`6`
`7`		`-interface ParentIf {`
	`7`	`+interface ParentIfK {`
`8`	`8`
`9`	`9`	`fun inheritedInterfaceMethodK()`
`10`	`10`
`11`	`11`	`}`
`12`	`12`
`13`		`-interface ChildIf : ParentIf {`
	`13`	`+interface ChildIfK : ParentIfK {`
`14`	`14`
`15`	`15`
`16`	`16`	`}`
`@@ -24,7 +24,7 @@ class ChildKt : TestKt() {`
`24`	`24`	`c.equals(c)`
`25`	`25`	`c.hashCode()`
`26`	`26`	`c.inheritMe()`
`27`		`- val c2: ParentIf? = null`
	`27`	`+ val c2: ParentIfK? = null`
`28`	`28`	`c2?.inheritedInterfaceMethodK()`
`29`	`29`
`30`	`30`	`}`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+\| test.kt:20:1:26:1 \| Foo \| 7 \| 6 \| 1 \|`