Skip to content

Commit 69cfc79

Browse files
committed
Kotlin: Add LighterAST support to numlines extraction
1 parent 257fe1a commit 69cfc79

File tree

6 files changed

+316
-123
lines changed

6 files changed

+316
-123
lines changed

java/kotlin-extractor/src/main/kotlin/KotlinFileExtractor.kt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -133,7 +133,7 @@ open class KotlinFileExtractor(
133133
val lighterAstCommentsExtracted = CommentExtractorLighterAST(this, file, tw.fileId).extract()
134134
if (psiCommentsExtracted == lighterAstCommentsExtracted) {
135135
if (psiCommentsExtracted) {
136-
logger.warnElement("Found both PSI and LightAST comments in ${file.path}.", file)
136+
logger.warnElement("Found both PSI and LighterAST comments in ${file.path}.", file)
137137
} else {
138138
logger.warnElement("Comments could not be processed in ${file.path}.", file)
139139
}
Lines changed: 10 additions & 120 deletions
Original file line numberDiff line numberDiff line change
@@ -1,138 +1,28 @@
11
package com.github.codeql
22

3-
import com.github.codeql.utils.versions.getPsi2Ir
4-
import com.intellij.psi.PsiComment
5-
import com.intellij.psi.PsiElement
6-
import com.intellij.psi.PsiWhiteSpace
7-
import org.jetbrains.kotlin.config.KotlinCompilerVersion
8-
import org.jetbrains.kotlin.ir.IrElement
93
import org.jetbrains.kotlin.ir.declarations.*
10-
import org.jetbrains.kotlin.kdoc.psi.api.KDocElement
11-
import org.jetbrains.kotlin.psi.KtCodeFragment
12-
import org.jetbrains.kotlin.psi.KtVisitor
134

145
class LinesOfCode(
156
val logger: FileLogger,
167
val tw: FileTrapWriter,
178
val file: IrFile
189
) {
19-
val psi2Ir = getPsi2Ir().also {
20-
if (it == null) {
21-
logger.warn("Lines of code will not be populated as Kotlin version is too old (${KotlinCompilerVersion.getVersion()})")
22-
}
23-
}
10+
val linesOfCodePSI = LinesOfCodePSI(logger, tw, file)
11+
val linesOfCodeLighterAST = LinesOfCodeLighterAST(logger, tw, file)
2412

2513
fun linesOfCodeInFile(id: Label<DbFile>) {
26-
if (psi2Ir == null) {
27-
return
14+
val psiExtracted = linesOfCodePSI.linesOfCodeInFile(id)
15+
val lighterASTExtracted = linesOfCodeLighterAST.linesOfCodeInFile(id)
16+
if (psiExtracted && lighterASTExtracted) {
17+
logger.warnElement("Both PSI and LighterAST number-of-lines-in-file information for ${file.path}.", file)
2818
}
29-
val ktFile = psi2Ir.getKtFile(file)
30-
if (ktFile == null) {
31-
return
32-
}
33-
linesOfCodeInPsi(id, ktFile, file)
3419
}
3520

3621
fun linesOfCodeInDeclaration(d: IrDeclaration, id: Label<out DbSourceline>) {
37-
if (psi2Ir == null) {
38-
return
39-
}
40-
val p = psi2Ir.findPsiElement(d, file)
41-
if (p == null) {
42-
return
43-
}
44-
linesOfCodeInPsi(id, p, d)
45-
}
46-
47-
private fun linesOfCodeInPsi(id: Label<out DbSourceline>, root: PsiElement, e: IrElement) {
48-
val document = root.getContainingFile().getViewProvider().getDocument()
49-
if (document == null) {
50-
logger.errorElement("Cannot find document for PSI", e)
51-
tw.writeNumlines(id, 0, 0, 0)
52-
return
22+
val psiExtracted = linesOfCodePSI.linesOfCodeInDeclaration(d, id)
23+
val lighterASTExtracted = linesOfCodeLighterAST.linesOfCodeInDeclaration(d, id)
24+
if (psiExtracted && lighterASTExtracted) {
25+
logger.warnElement("Both PSI and LighterAST number-of-lines-in-file information for declaration.", d)
5326
}
54-
55-
val rootRange = root.getTextRange()
56-
val rootFirstLine = document.getLineNumber(rootRange.getStartOffset())
57-
val rootLastLine = document.getLineNumber(rootRange.getEndOffset())
58-
if (rootLastLine < rootFirstLine) {
59-
logger.errorElement("PSI ends before it starts", e)
60-
tw.writeNumlines(id, 0, 0, 0)
61-
return
62-
}
63-
val numLines = 1 + rootLastLine - rootFirstLine
64-
val lineContents = Array(numLines) { LineContent() }
65-
66-
val visitor =
67-
object : KtVisitor<Unit, Unit>() {
68-
override fun visitElement(element: PsiElement) {
69-
val isComment = element is PsiComment
70-
// Comments may include nodes that aren't PsiComments,
71-
// so we don't want to visit them or we'll think they
72-
// are code.
73-
if (!isComment) {
74-
element.acceptChildren(this)
75-
}
76-
77-
if (element is PsiWhiteSpace) {
78-
return
79-
}
80-
// Leaf nodes are assumed to be tokens, and
81-
// therefore we count any lines that they are on.
82-
// For comments, we actually need to look at the
83-
// outermost node, as the leaves of KDocs don't
84-
// necessarily cover all lines.
85-
if (isComment || element.getChildren().size == 0) {
86-
val range = element.getTextRange()
87-
val startOffset = range.getStartOffset()
88-
val endOffset = range.getEndOffset()
89-
// The PSI doesn't seem to have anything like
90-
// the IR's UNDEFINED_OFFSET and SYNTHETIC_OFFSET,
91-
// but < 0 still seem to represent bad/unknown
92-
// locations.
93-
if (startOffset < 0 || endOffset < 0) {
94-
logger.errorElement("PSI has negative offset", e)
95-
return
96-
}
97-
if (startOffset > endOffset) {
98-
return
99-
}
100-
// We might get e.g. an import list for a file
101-
// with no imports, which claims to have start
102-
// and end offsets of 0. Anything of 0 width
103-
// we therefore just skip.
104-
if (startOffset == endOffset) {
105-
return
106-
}
107-
val firstLine = document.getLineNumber(startOffset)
108-
val lastLine = document.getLineNumber(endOffset)
109-
if (firstLine < rootFirstLine) {
110-
logger.errorElement("PSI element starts before root", e)
111-
return
112-
} else if (lastLine > rootLastLine) {
113-
logger.errorElement("PSI element ends after root", e)
114-
return
115-
}
116-
for (line in firstLine..lastLine) {
117-
val lineContent = lineContents[line - rootFirstLine]
118-
if (isComment) {
119-
lineContent.containsComment = true
120-
} else {
121-
lineContent.containsCode = true
122-
}
123-
}
124-
}
125-
}
126-
}
127-
root.accept(visitor)
128-
val total = lineContents.size
129-
val code = lineContents.count { it.containsCode }
130-
val comment = lineContents.count { it.containsComment }
131-
tw.writeNumlines(id, total, code, comment)
132-
}
133-
134-
private class LineContent {
135-
var containsComment = false
136-
var containsCode = false
13727
}
13828
}
Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
package com.github.codeql
2+
3+
import com.github.codeql.utils.versions.getPsi2Ir
4+
import com.intellij.psi.PsiComment
5+
import com.intellij.psi.PsiElement
6+
import com.intellij.psi.PsiWhiteSpace
7+
import org.jetbrains.kotlin.config.KotlinCompilerVersion
8+
import org.jetbrains.kotlin.ir.IrElement
9+
import org.jetbrains.kotlin.ir.declarations.*
10+
import org.jetbrains.kotlin.kdoc.psi.api.KDocElement
11+
import org.jetbrains.kotlin.psi.KtCodeFragment
12+
import org.jetbrains.kotlin.psi.KtVisitor
13+
14+
class LinesOfCodePSI(
15+
val logger: FileLogger,
16+
val tw: FileTrapWriter,
17+
val file: IrFile
18+
) {
19+
val psi2Ir = getPsi2Ir().also {
20+
if (it == null) {
21+
logger.warn("Lines of code will not be populated as Kotlin version is too old (${KotlinCompilerVersion.getVersion()})")
22+
}
23+
}
24+
25+
fun linesOfCodeInFile(id: Label<DbFile>): Boolean {
26+
if (psi2Ir == null) {
27+
return false
28+
}
29+
val ktFile = psi2Ir.getKtFile(file)
30+
if (ktFile == null) {
31+
return false
32+
}
33+
linesOfCodeInPsi(id, ktFile, file)
34+
// Even if linesOfCodeInPsi didn't manage to extract any
35+
// information, if we got as far as calling it then we have
36+
// PSI info for the file
37+
return true
38+
}
39+
40+
fun linesOfCodeInDeclaration(d: IrDeclaration, id: Label<out DbSourceline>): Boolean {
41+
if (psi2Ir == null) {
42+
return false
43+
}
44+
val p = psi2Ir.findPsiElement(d, file)
45+
if (p == null) {
46+
return false
47+
}
48+
linesOfCodeInPsi(id, p, d)
49+
// Even if linesOfCodeInPsi didn't manage to extract any
50+
// information, if we got as far as calling it then we have
51+
// PSI info for the declaration
52+
return true
53+
}
54+
55+
private fun linesOfCodeInPsi(id: Label<out DbSourceline>, root: PsiElement, e: IrElement) {
56+
val document = root.getContainingFile().getViewProvider().getDocument()
57+
if (document == null) {
58+
logger.errorElement("Cannot find document for PSI", e)
59+
tw.writeNumlines(id, 0, 0, 0)
60+
return
61+
}
62+
63+
val rootRange = root.getTextRange()
64+
val rootStartOffset = rootRange.getStartOffset()
65+
val rootEndOffset = rootRange.getEndOffset()
66+
if (rootStartOffset < 0 || rootEndOffset < 0) {
67+
// This is synthetic, or has an invalid location
68+
tw.writeNumlines(id, 0, 0, 0)
69+
return
70+
}
71+
val rootFirstLine = document.getLineNumber(rootStartOffset)
72+
val rootLastLine = document.getLineNumber(rootEndOffset)
73+
if (rootLastLine < rootFirstLine) {
74+
logger.errorElement("PSI ends before it starts", e)
75+
tw.writeNumlines(id, 0, 0, 0)
76+
return
77+
}
78+
val numLines = 1 + rootLastLine - rootFirstLine
79+
val lineContents = Array(numLines) { LineContent() }
80+
81+
val visitor =
82+
object : KtVisitor<Unit, Unit>() {
83+
override fun visitElement(element: PsiElement) {
84+
val isComment = element is PsiComment
85+
// Comments may include nodes that aren't PsiComments,
86+
// so we don't want to visit them or we'll think they
87+
// are code.
88+
if (!isComment) {
89+
element.acceptChildren(this)
90+
}
91+
92+
if (element is PsiWhiteSpace) {
93+
return
94+
}
95+
// Leaf nodes are assumed to be tokens, and
96+
// therefore we count any lines that they are on.
97+
// For comments, we actually need to look at the
98+
// outermost node, as the leaves of KDocs don't
99+
// necessarily cover all lines.
100+
if (isComment || element.getChildren().size == 0) {
101+
val range = element.getTextRange()
102+
val startOffset = range.getStartOffset()
103+
val endOffset = range.getEndOffset()
104+
// The PSI doesn't seem to have anything like
105+
// the IR's UNDEFINED_OFFSET and SYNTHETIC_OFFSET,
106+
// but < 0 still seem to represent bad/unknown
107+
// locations.
108+
if (startOffset < 0 || endOffset < 0) {
109+
logger.errorElement("PSI element has negative offset", e)
110+
return
111+
}
112+
if (startOffset > endOffset) {
113+
logger.errorElement("PSI element has negative size", e)
114+
return
115+
}
116+
// We might get e.g. an import list for a file
117+
// with no imports, which claims to have start
118+
// and end offsets of 0. Anything of 0 width
119+
// we therefore just skip.
120+
if (startOffset == endOffset) {
121+
return
122+
}
123+
val firstLine = document.getLineNumber(startOffset)
124+
val lastLine = document.getLineNumber(endOffset)
125+
if (firstLine < rootFirstLine) {
126+
logger.errorElement("PSI element starts before root", e)
127+
return
128+
} else if (lastLine > rootLastLine) {
129+
logger.errorElement("PSI element ends after root", e)
130+
return
131+
}
132+
for (line in firstLine..lastLine) {
133+
val lineContent = lineContents[line - rootFirstLine]
134+
if (isComment) {
135+
lineContent.containsComment = true
136+
} else {
137+
lineContent.containsCode = true
138+
}
139+
}
140+
}
141+
}
142+
}
143+
root.accept(visitor)
144+
val code = lineContents.count { it.containsCode }
145+
val comment = lineContents.count { it.containsComment }
146+
tw.writeNumlines(id, numLines, code, comment)
147+
}
148+
149+
private class LineContent {
150+
var containsComment = false
151+
var containsCode = false
152+
}
153+
}
Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
package com.github.codeql
2+
3+
import org.jetbrains.kotlin.ir.declarations.*
4+
5+
class LinesOfCodeLighterAST(
6+
val logger: FileLogger,
7+
val tw: FileTrapWriter,
8+
val file: IrFile
9+
) {
10+
// We don't support LighterAST with old Kotlin versions
11+
fun linesOfCodeInFile(@Suppress("UNUSED_PARAMETER") id: Label<DbFile>): Boolean {
12+
return false
13+
}
14+
15+
// We don't support LighterAST with old Kotlin versions
16+
fun linesOfCodeInDeclaration(@Suppress("UNUSED_PARAMETER") d: IrDeclaration, @Suppress("UNUSED_PARAMETER") id: Label<out DbSourceline>): Boolean {
17+
return false
18+
}
19+
}

java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_9_0-Beta/CommentExtractorLighterAST.kt

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,6 @@ import org.jetbrains.kotlin.ir.visitors.acceptVoid
1313
import org.jetbrains.kotlin.ir.visitors.IrElementVisitorVoid
1414
import org.jetbrains.kotlin.kdoc.lexer.KDocTokens
1515
import org.jetbrains.kotlin.lexer.KtTokens
16-
import org.jetbrains.kotlin.psi.psiUtil.endOffset
17-
import org.jetbrains.kotlin.psi.psiUtil.startOffset
1816
import org.jetbrains.kotlin.util.getChildren
1917

2018
class CommentExtractorLighterAST(fileExtractor: KotlinFileExtractor, file: IrFile, fileLabel: Label<out DbFile>): CommentExtractor(fileExtractor, file, fileLabel) {

0 commit comments

Comments
 (0)