Skip to content

Commit 08e3431

Browse files
smowtonsmowton
committed
Also stub class files relating to file classes and top-level declarations
1 parent 748637c commit 08e3431

File tree

5 files changed

+105
-70
lines changed

5 files changed

+105
-70
lines changed

‎java/kotlin-extractor/src/main/java/com/semmle/extractor/java/OdasaOutput.java

Lines changed: 28 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,10 @@
1919
import java.util.zip.ZipFile;
2020

2121
import com.github.codeql.Logger;
22-
import static com.github.codeql.ClassNamesKt.getIrDeclBinaryName;
22+
import static com.github.codeql.ClassNamesKt.getIrElementBinaryName;
2323
import static com.github.codeql.ClassNamesKt.getIrClassVirtualFile;
2424

25+
import org.jetbrains.kotlin.ir.IrElement;
2526
import org.jetbrains.kotlin.ir.declarations.IrClass;
2627

2728
import com.intellij.openapi.vfs.VirtualFile;
@@ -212,20 +213,19 @@ private File trapFileFor(File file) {
212213
PathTransformer.std().fileAsDatabaseString(file) + ".trap.gz");
213214
}
214215

215-
private File getTrapFileForDecl(IrDeclaration sym, String signature) {
216+
private File getTrapFileForDecl(IrElement sym, String signature) {
216217
if (currentSpecFileEntry == null)
217218
return null;
218219
return trapFileForDecl(sym, signature);
219220
}
220221

221-
private File trapFileForDecl(IrDeclaration sym, String signature) {
222+
private File trapFileForDecl(IrElement sym, String signature) {
222223
return FileUtil.fileRelativeTo(currentSpecFileEntry.getTrapFolder(),
223224
trapFilePathForDecl(sym, signature));
224225
}
225226

226-
private String trapFilePathForDecl(IrDeclaration sym, String signature) {
227-
String binaryName = getIrDeclBinaryName(sym);
228-
String binaryNameWithSignature = binaryName + signature;
227+
private String trapFilePathForDecl(IrElement sym, String signature) {
228+
String binaryName = getIrElementBinaryName(sym);
229229
// TODO: Reinstate this?
230230
//if (getTrackClassOrigins())
231231
// classId += "-" + StringDigestor.digest(sym.getSourceFileId());
@@ -241,7 +241,7 @@ private String trapFilePathForDecl(IrDeclaration sym, String signature) {
241241
* Deletion of existing trap files.
242242
*/
243243

244-
private void deleteTrapFileAndDependencies(IrDeclaration sym, String signature) {
244+
private void deleteTrapFileAndDependencies(IrElement sym, String signature) {
245245
File trap = trapFileForDecl(sym, signature);
246246
if (trap.exists()) {
247247
trap.delete();
@@ -269,7 +269,7 @@ private void deleteTrapFileAndDependencies(IrDeclaration sym, String signature)
269269
* Any unique suffix needed to distinguish `sym` from other declarations with the same name.
270270
* For functions for example, this means its parameter signature.
271271
*/
272-
private TrapFileManager getMembersWriterForDecl(File trap, File trapFileBase, TrapClassVersion trapFileVersion, IrDeclaration sym, String signature) {
272+
private TrapFileManager getMembersWriterForDecl(File trap, File trapFileBase, TrapClassVersion trapFileVersion, IrElement sym, String signature) {
273273
if (use_trap_locking) {
274274
TrapClassVersion currVersion = TrapClassVersion.fromSymbol(sym, log);
275275
String shortName = sym instanceof IrDeclarationWithName ? ((IrDeclarationWithName)sym).getName().asString() : "(name unknown)";
@@ -326,7 +326,7 @@ private TrapFileManager getMembersWriterForDecl(File trap, File trapFileBase, Tr
326326
return trapWriter(trap, sym, signature);
327327
}
328328

329-
private TrapFileManager trapWriter(File trapFile, IrDeclaration sym, String signature) {
329+
private TrapFileManager trapWriter(File trapFile, IrElement sym, String signature) {
330330
if (!trapFile.getName().endsWith(".trap.gz"))
331331
throw new CatastrophicError("OdasaOutput only supports writing to compressed trap files");
332332
String relative = FileUtil.relativePath(trapFile, currentSpecFileEntry.getTrapFolder());
@@ -335,7 +335,7 @@ private TrapFileManager trapWriter(File trapFile, IrDeclaration sym, String sign
335335
return concurrentWriter(trapFile, relative, log, sym, signature);
336336
}
337337

338-
private TrapFileManager concurrentWriter(File trapFile, String relative, Logger log, IrDeclaration sym, String signature) {
338+
private TrapFileManager concurrentWriter(File trapFile, String relative, Logger log, IrElement sym, String signature) {
339339
if (trapFile.exists())
340340
return null;
341341
return new TrapFileManager(trapFile, relative, true, log, sym, signature);
@@ -345,11 +345,11 @@ public class TrapFileManager implements AutoCloseable {
345345

346346
private TrapDependencies trapDependenciesForClass;
347347
private File trapFile;
348-
private IrDeclaration sym;
348+
private IrElement sym;
349349
private String signature;
350350
private boolean hasError = false;
351351

352-
private TrapFileManager(File trapFile, String relative, boolean concurrentCreation, Logger log, IrDeclaration sym, String signature) {
352+
private TrapFileManager(File trapFile, String relative, boolean concurrentCreation, Logger log, IrElement sym, String signature) {
353353
trapDependenciesForClass = new TrapDependencies(relative);
354354
this.trapFile = trapFile;
355355
this.sym = sym;
@@ -360,7 +360,7 @@ public File getFile() {
360360
return trapFile;
361361
}
362362

363-
public void addDependency(IrDeclaration dep, String signature) {
363+
public void addDependency(IrElement dep, String signature) {
364364
trapDependenciesForClass.addDependency(trapFilePathForDecl(dep, signature));
365365
}
366366

@@ -460,19 +460,19 @@ public TrapLocker getTrapLockerForModule(String moduleName) {
460460
*
461461
* @return a {@link TrapLocker} for the trap file corresponding to the given class symbol.
462462
*/
463-
public TrapLocker getTrapLockerForDecl(IrDeclaration sym, String signature, boolean fromSource) {
463+
public TrapLocker getTrapLockerForDecl(IrElement sym, String signature, boolean fromSource) {
464464
return new TrapLocker(sym, signature, fromSource);
465465
}
466466

467467
public class TrapLocker implements AutoCloseable {
468-
private final IrDeclaration sym;
468+
private final IrElement sym;
469469
private final File trapFile;
470470
// trapFileBase is used when doing lockless TRAP file writing.
471471
// It is trapFile without the #metadata.trap.gz suffix.
472472
private File trapFileBase = null;
473473
private TrapClassVersion trapFileVersion = null;
474474
private final String signature;
475-
private TrapLocker(IrDeclaration decl, String signature, boolean fromSource) {
475+
private TrapLocker(IrElement decl, String signature, boolean fromSource) {
476476
this.sym = decl;
477477
this.signature = signature;
478478
if (sym==null) {
@@ -720,11 +720,18 @@ private static long getVirtualFileTimeStamp(VirtualFile vf, Logger log) {
720720
return vf.getTimeStamp();
721721
}
722722

723-
private static TrapClassVersion fromSymbol(IrDeclaration sym, Logger log) {
724-
VirtualFile vf = sym instanceof IrClass ? getIrClassVirtualFile((IrClass)sym) :
725-
sym.getParent() instanceof IrClass ? getIrClassVirtualFile((IrClass)sym.getParent()) :
726-
null;
727-
if(vf == null)
723+
private static VirtualFile getVirtualFileIfClass(IrElement e) {
724+
if (e instanceof IrClass)
725+
return getIrClassVirtualFile((IrClass)e);
726+
else
727+
return null;
728+
}
729+
730+
private static TrapClassVersion fromSymbol(IrElement sym, Logger log) {
731+
VirtualFile vf = getVirtualFileIfClass(sym);
732+
if (vf == null && sym instanceof IrDeclaration)
733+
vf = getVirtualFileIfClass(((IrDeclaration)sym).getParent());
734+
if (vf == null)
728735
return new TrapClassVersion(-1, 0, 0, null);
729736

730737
final int[] versionStore = new int[1];

‎java/kotlin-extractor/src/main/kotlin/ExternalDeclExtractor.kt

Lines changed: 10 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,13 @@
11
package com.github.codeql
22

3-
import com.github.codeql.utils.isExternalDeclaration
43
import com.github.codeql.utils.isExternalFileClassMember
54
import com.semmle.extractor.java.OdasaOutput
65
import com.semmle.util.data.StringDigestor
76
import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext
7+
import org.jetbrains.kotlin.ir.IrElement
88
import org.jetbrains.kotlin.ir.declarations.*
9-
import org.jetbrains.kotlin.ir.util.fqNameWhenAvailable
109
import org.jetbrains.kotlin.ir.util.isFileClass
1110
import org.jetbrains.kotlin.ir.util.packageFqName
12-
import org.jetbrains.kotlin.ir.util.parentClassOrNull
13-
import org.jetbrains.kotlin.name.FqName
1411
import java.io.BufferedWriter
1512
import java.io.File
1613
import java.util.ArrayList
@@ -35,30 +32,30 @@ class ExternalDeclExtractor(val logger: FileLogger, val invocationTrapFile: Stri
3532
logger.errorElement("External declaration is neither a class, nor a top-level declaration", d)
3633
return false
3734
}
38-
val declBinaryName = declBinaryNames.getOrPut(d) { getIrDeclBinaryName(d) }
35+
val declBinaryName = declBinaryNames.getOrPut(d) { getIrElementBinaryName(d) }
3936
val ret = externalDeclsDone.add(Pair(declBinaryName, signature))
4037
if (ret) externalDeclWorkList.add(Pair(d, signature))
4138
return ret
4239
}
4340
fun extractLater(c: IrClass) = extractLater(c, "")
4441

45-
fun noteClassSourceExtractedTo(c: IrClass, sourceFile: String) {
46-
extractDecl(c, "", true) { trapFileBW, _, _ ->
47-
trapFileBW.write("// .class trap file stubbed because this class was extracted from source in $sourceFile\n")
42+
fun noteElementExtractedFromSource(e: IrElement, signature: String = "") {
43+
extractElement(e, signature, true) { trapFileBW, _, _ ->
44+
trapFileBW.write("// Trap file stubbed because this declaration was extracted from source in $sourceFilePath\n")
4845
trapFileBW.write("// Part of invocation $invocationTrapFile\n")
4946
}
5047
}
5148

52-
private fun extractDecl(irDecl: IrDeclaration, possiblyLongSignature: String, fromSource: Boolean, extractorFn: (BufferedWriter, String, OdasaOutput.TrapFileManager) -> Unit) {
49+
private fun extractElement(element: IrElement, possiblyLongSignature: String, fromSource: Boolean, extractorFn: (BufferedWriter, String, OdasaOutput.TrapFileManager) -> Unit) {
5350
// In order to avoid excessively long signatures which can lead to trap file names longer than the filesystem
5451
// limit, we truncate and add a hash to preserve uniqueness if necessary.
5552
val signature = if (possiblyLongSignature.length > 100) {
5653
possiblyLongSignature.substring(0, 92) + "#" + StringDigestor.digest(possiblyLongSignature).substring(0, 8)
5754
} else { possiblyLongSignature }
58-
output.getTrapLockerForDecl(irDecl, signature, fromSource).useAC { locker ->
55+
output.getTrapLockerForDecl(element, signature, fromSource).useAC { locker ->
5956
locker.trapFileManager.useAC { manager ->
60-
val shortName = when(irDecl) {
61-
is IrDeclarationWithName -> irDecl.name.asString()
57+
val shortName = when(element) {
58+
is IrDeclarationWithName -> element.name.asString()
6259
else -> "(unknown name)"
6360
}
6461
if (manager == null) {
@@ -89,7 +86,7 @@ class ExternalDeclExtractor(val logger: FileLogger, val invocationTrapFile: Stri
8986
externalDeclWorkList.clear()
9087
nextBatch.forEach { workPair ->
9188
val (irDecl, possiblyLongSignature) = workPair
92-
extractDecl(irDecl, possiblyLongSignature, false) { trapFileBW, signature, manager ->
89+
extractElement(irDecl, possiblyLongSignature, false) { trapFileBW, signature, manager ->
9390
val containingClass = getContainingClassOrSelf(irDecl)
9491
if (containingClass == null) {
9592
logger.errorElement("Unable to get containing class", irDecl)

‎java/kotlin-extractor/src/main/kotlin/KotlinFileExtractor.kt

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,12 @@ open class KotlinFileExtractor(
9090
}
9191
}
9292

93-
file.declarations.forEach { extractDeclaration(it, extractPrivateMembers = true, extractFunctionBodies = true) }
93+
file.declarations.forEach {
94+
extractDeclaration(it, extractPrivateMembers = true, extractFunctionBodies = true)
95+
if (it !is IrClass) {
96+
externalClassExtractor.noteElementExtractedFromSource(it, getTrapFileSignature(it))
97+
}
98+
}
9499
extractStaticInitializer(file, { extractFileClass(file) })
95100
CommentExtractor(this, file, tw.fileId).extract()
96101

@@ -99,6 +104,8 @@ open class KotlinFileExtractor(
99104
}
100105

101106
linesOfCode?.linesOfCodeInFile(id)
107+
108+
externalClassExtractor.noteElementExtractedFromSource(file)
102109
}
103110
}
104111

@@ -517,7 +524,7 @@ open class KotlinFileExtractor(
517524
linesOfCode?.linesOfCodeInDeclaration(c, id)
518525

519526
if (extractFunctionBodies)
520-
externalClassExtractor.noteClassSourceExtractedTo(c, tw.filePath)
527+
externalClassExtractor.noteElementExtractedFromSource(c)
521528

522529
return id
523530
}

‎java/kotlin-extractor/src/main/kotlin/KotlinUsesExtractor.kt

Lines changed: 22 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -69,13 +69,11 @@ open class KotlinUsesExtractor(
6969

7070
@OptIn(kotlin.ExperimentalStdlibApi::class) // Annotation required by kotlin versions < 1.5
7171
fun extractFileClass(f: IrFile): Label<out DbClass> {
72-
val fileName = f.fileEntry.name
7372
val pkg = f.fqName.asString()
74-
val defaultName = fileName.replaceFirst(Regex(""".*[/\\]"""), "").replaceFirst(Regex("""\.kt$"""), "").replaceFirstChar({ it.uppercase() }) + "Kt"
75-
var jvmName = getJvmName(f) ?: defaultName
73+
val jvmName = getFileClassName(f)
7674
val qualClassName = if (pkg.isEmpty()) jvmName else "$pkg.$jvmName"
7775
val label = "@\"class;$qualClassName\""
78-
val id: Label<DbClass> = tw.getLabelFor(label, {
76+
val id: Label<DbClass> = tw.getLabelFor(label) {
7977
val fileId = tw.mkFileId(f.path, false)
8078
val locId = tw.getWholeFileLocation(fileId)
8179
val pkgId = extractPackage(pkg)
@@ -84,7 +82,7 @@ open class KotlinUsesExtractor(
8482
tw.writeHasLocation(it, locId)
8583

8684
addModifiers(it, "public", "final")
87-
})
85+
}
8886
return id
8987
}
9088

@@ -258,10 +256,26 @@ open class KotlinUsesExtractor(
258256
private fun propertySignature(p: IrProperty) =
259257
((p.getter ?: p.setter)?.extensionReceiverParameter?.let { useType(erase(it.type)).javaResult.signature } ?: "")
260258

259+
fun getTrapFileSignature(d: IrDeclaration) =
260+
when(d) {
261+
is IrFunction ->
262+
// Note we erase the parameter types before calling useType even though the signature should be the same
263+
// in order to prevent an infinite loop through useTypeParameter -> useDeclarationParent -> useFunction
264+
// -> extractFunctionLaterIfExternalFileMember, which would result for `fun <T> f(t: T) { ... }` for example.
265+
(listOfNotNull(d.extensionReceiverParameter) + d.valueParameters)
266+
.map { useType(erase(it.type)).javaResult.signature }
267+
.joinToString(separator = ",", prefix = "(", postfix = ")")
268+
is IrProperty -> propertySignature(d) + externalClassExtractor.propertySignature
269+
is IrField -> (d.correspondingPropertySymbol?.let { propertySignature(it.owner) } ?: "") + externalClassExtractor.fieldSignature
270+
else -> "unknown signature".also {
271+
logger.warn("Trap file signature requested for unexpected element $d")
272+
}
273+
}
274+
261275
private fun extractPropertyLaterIfExternalFileMember(p: IrProperty) {
262276
if (isExternalFileClassMember(p)) {
263277
extractExternalClassLater(p.parentAsClass)
264-
val signature = propertySignature(p) + externalClassExtractor.propertySignature
278+
val signature = getTrapFileSignature(p)
265279
dependencyCollector?.addDependency(p, signature)
266280
externalClassExtractor.extractLater(p, signature)
267281
}
@@ -270,7 +284,7 @@ open class KotlinUsesExtractor(
270284
private fun extractFieldLaterIfExternalFileMember(f: IrField) {
271285
if (isExternalFileClassMember(f)) {
272286
extractExternalClassLater(f.parentAsClass)
273-
val signature = (f.correspondingPropertySymbol?.let { propertySignature(it.owner) } ?: "") + externalClassExtractor.fieldSignature
287+
val signature = getTrapFileSignature(f)
274288
dependencyCollector?.addDependency(f, signature)
275289
externalClassExtractor.extractLater(f, signature)
276290
}
@@ -285,18 +299,7 @@ open class KotlinUsesExtractor(
285299
// getters and setters are extracted alongside it
286300
return
287301
}
288-
// Note we erase the parameter types before calling useType even though the signature should be the same
289-
// in order to prevent an infinite loop through useTypeParameter -> useDeclarationParent -> useFunction
290-
// -> extractFunctionLaterIfExternalFileMember, which would result for `fun <T> f(t: T) { ... }` for example.
291-
val ext = f.extensionReceiverParameter
292-
val parameters = if (ext != null) {
293-
listOf(ext) + f.valueParameters
294-
} else {
295-
f.valueParameters
296-
}
297-
298-
val paramSigs = parameters.map { useType(erase(it.type)).javaResult.signature }
299-
val signature = paramSigs.joinToString(separator = ",", prefix = "(", postfix = ")")
302+
val signature = getTrapFileSignature(f)
300303
dependencyCollector?.addDependency(f, signature)
301304
externalClassExtractor.extractLater(f, signature)
302305
}

‎java/kotlin-extractor/src/main/kotlin/utils/ClassNames.kt

Lines changed: 36 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import org.jetbrains.kotlin.load.kotlin.KotlinJvmBinarySourceElement
99

1010
import com.intellij.openapi.vfs.VirtualFile
1111
import org.jetbrains.kotlin.builtins.jvm.JavaToKotlinClassMap
12+
import org.jetbrains.kotlin.ir.IrElement
1213
import org.jetbrains.kotlin.ir.declarations.*
1314
import org.jetbrains.kotlin.ir.util.fqNameWhenAvailable
1415
import org.jetbrains.kotlin.ir.util.parentClassOrNull
@@ -21,20 +22,40 @@ import org.jetbrains.kotlin.load.kotlin.JvmPackagePartSource
2122
// for `that`.
2223
private fun getName(d: IrDeclarationWithName) = (d as? IrAnnotationContainer)?.let { getJvmName(it) } ?: d.name.asString()
2324

24-
fun getIrDeclBinaryName(that: IrDeclaration): String {
25-
val shortName = when(that) {
26-
is IrDeclarationWithName -> getName(that)
27-
else -> "(unknown-name)"
28-
}
29-
val internalName = StringBuilder(shortName);
30-
generateSequence(that.parent) { (it as? IrDeclaration)?.parent }
31-
.forEach {
32-
when (it) {
33-
is IrClass -> internalName.insert(0, getName(it) + "$")
34-
is IrPackageFragment -> it.fqName.asString().takeIf { fqName -> fqName.isNotEmpty() }?.let { fqName -> internalName.insert(0, "$fqName.") }
35-
}
36-
}
37-
return internalName.toString()
25+
fun getFileClassName(f: IrFile) =
26+
getJvmName(f) ?:
27+
((f.fileEntry.name.replaceFirst(Regex(""".*[/\\]"""), "")
28+
.replaceFirst(Regex("""\.kt$"""), "")
29+
.replaceFirstChar { it.uppercase() }) + "Kt")
30+
31+
fun getIrElementBinaryName(that: IrElement): String {
32+
if (that is IrFile) {
33+
val shortName = getFileClassName(that)
34+
val pkg = that.fqName.asString()
35+
return if (pkg.isEmpty()) shortName else "$pkg.$shortName"
36+
}
37+
38+
if (that !is IrDeclaration) {
39+
return "(unknown-name)"
40+
}
41+
42+
val shortName = when(that) {
43+
is IrDeclarationWithName -> getName(that)
44+
else -> "(unknown-name)"
45+
}
46+
val internalName = StringBuilder(shortName)
47+
generateSequence(that.parent) { (it as? IrDeclaration)?.parent }
48+
.forEach {
49+
if (it is IrFile) {
50+
// Note we'll fall through and do the IrPackageFragment case as well, since IrFile <: IrPackageFragment
51+
internalName.insert(0, getFileClassName(it) + "$")
52+
}
53+
when (it) {
54+
is IrClass -> internalName.insert(0, getName(it) + "$")
55+
is IrPackageFragment -> it.fqName.asString().takeIf { fqName -> fqName.isNotEmpty() }?.let { fqName -> internalName.insert(0, "$fqName.") }
56+
}
57+
}
58+
return internalName.toString()
3859
}
3960

4061
fun getIrClassVirtualFile(irClass: IrClass): VirtualFile? {
@@ -81,7 +102,7 @@ private fun getRawIrClassBinaryPath(irClass: IrClass) =
81102
fun getIrClassBinaryPath(irClass: IrClass): String {
82103
return getRawIrClassBinaryPath(irClass)
83104
// Otherwise, make up a fake location:
84-
?: "/!unknown-binary-location/${getIrDeclBinaryName(irClass).replace(".", "/")}.class"
105+
?: "/!unknown-binary-location/${getIrElementBinaryName(irClass).replace(".", "/")}.class"
85106
}
86107

87108
fun getContainingClassOrSelf(decl: IrDeclaration): IrClass? {

0 commit comments

Comments
 (0)