[jssrc2cpg] Use ForkJoinParallelCpgPassWithAccumulator (#5883)

Author: max-leuthaeuser

build.sbt

@@ -2,7 +2,7 @@ name                     := "joern"
 ThisBuild / organization := "io.joern"
 ThisBuild / scalaVersion := "3.6.4"
 
-val cpgVersion = "1.7.58"
+val cpgVersion = "1.7.60"
 
 lazy val joerncli          = Projects.joerncli
 lazy val querydb           = Projects.querydb

joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/CGlobal.scala

@@ -19,8 +19,8 @@ class CGlobal extends Global {
   // 4) in this map with value HeaderFileParserLanguage.Both -> included from at least one C and one C++ source file, so we will have to parse it twice
   val headerIncludes: ConcurrentHashMap[String, HeaderFileParserLanguage] = new ConcurrentHashMap()
 
-  def typesSeen(): List[String] = {
-    usedTypes.keys().asScala.toList
+  def typesSeen(): Set[String] = {
+    usedTypes.keys().asScala.toSet
   }
 
   def unhandledMethodDeclarations(): Map[String, MethodInfo] = {

joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/Ghidra2Cpg.scala

@@ -125,7 +125,7 @@ class Ghidra2Cpg extends X2CpgFrontend {
         new ReturnEdgesPass(cpg).createAndApply()
     }
 
-    TypeNodePass.withRegisteredTypes(Types.types.toList, cpg).createAndApply()
+    TypeNodePass.withRegisteredTypes(Types.types.toSet, cpg).createAndApply()
     new JumpPass(cpg).createAndApply()
     new LiteralPass(cpg, flatProgramAPI).createAndApply()
   }

joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/JavaSrc2Cpg.scala

@@ -29,7 +29,7 @@ class JavaSrc2Cpg extends X2CpgFrontend {
       new OuterClassRefPass(cpg).createAndApply()
       JavaConfigFileCreationPass(cpg, config = config).createAndApply()
       if (!config.skipTypeInfPass) {
-        TypeNodePass.withRegisteredTypes(astCreationPass.global.usedTypes.keys().asScala.toList, cpg).createAndApply()
+        TypeNodePass.withRegisteredTypes(astCreationPass.global.usedTypes.keys().asScala.toSet, cpg).createAndApply()
         new TypeInferencePass(cpg).createAndApply()
       }
     }

joern-cli/frontends/jimple2cpg/src/main/scala/io/joern/jimple2cpg/Jimple2Cpg.scala

@@ -126,7 +126,7 @@ class Jimple2Cpg extends X2CpgFrontend {
 
     val global = globalFromAstCreation()
     TypeNodePass
-      .withRegisteredTypes(global.usedTypes.keys().asScala.toList, cpg)
+      .withRegisteredTypes(global.usedTypes.keys().asScala.toSet, cpg)
       .createAndApply()
     DeclarationRefPass(cpg).createAndApply()
     JavaConfigFileCreationPass(cpg, Option(tmpDir.toString), config).createAndApply()

joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/astcreation/AstCreator.scala

@@ -7,7 +7,6 @@ import io.joern.jssrc2cpg.parser.BabelNodeInfo
 import io.joern.x2cpg.Ast
 import io.joern.x2cpg.AstCreatorBase
 import io.joern.x2cpg.ValidationMode
-import io.joern.x2cpg.datastructures.Global
 import io.joern.x2cpg.datastructures.Stack.*
 import io.joern.x2cpg.datastructures.VariableScopeManager
 import io.joern.x2cpg.frontendspecific.jssrc2cpg.Defines
@@ -26,7 +25,7 @@ import ujson.Value
 
 import scala.collection.mutable
 
-class AstCreator(val config: Config, val global: Global, val parserResult: ParseResult)(implicit
+class AstCreator(val config: Config, val usedTypes: mutable.HashSet[String], val parserResult: ParseResult)(implicit
   withSchemaValidation: ValidationMode
 ) extends AstCreatorBase[BabelNodeInfo, AstCreator](parserResult.filename)
     with AstForExpressionsCreator

joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/astcreation/AstCreatorHelper.scala

@@ -107,7 +107,7 @@ trait AstCreatorHelper(implicit withSchemaValidation: ValidationMode) { this: As
   }
 
   protected def registerType(typeFullName: String): Unit = {
-    global.usedTypes.putIfAbsent(typeFullName, true)
+    usedTypes.add(typeFullName)
   }
 
   protected def codeForNodes(nodes: Seq[NewNode]): Option[String] = nodes.collectFirst {

joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/passes/AstCreationPass.scala

@@ -5,31 +5,33 @@ import io.joern.jssrc2cpg.astcreation.AstCreator
 import io.joern.jssrc2cpg.parser.BabelJsonParser
 import io.joern.jssrc2cpg.utils.AstGenRunner.AstGenRunnerResult
 import io.joern.x2cpg.ValidationMode
-import io.joern.x2cpg.datastructures.Global
-import io.joern.x2cpg.frontendspecific.jssrc2cpg.Defines
-import io.joern.x2cpg.utils.Report
-import io.joern.x2cpg.utils.TimeUtils
+import io.joern.x2cpg.utils.{Report, TimeUtils}
 import io.shiftleft.codepropertygraph.generated.Cpg
-import io.shiftleft.passes.ForkJoinParallelCpgPass
+import io.shiftleft.passes.ForkJoinParallelCpgPassWithAccumulator
 import io.shiftleft.utils.IOUtils
-import org.slf4j.Logger
-import org.slf4j.LoggerFactory
+import org.slf4j.{Logger, LoggerFactory}
 
 import java.nio.file.Paths
-import scala.jdk.CollectionConverters.*
-import scala.util.Failure
-import scala.util.Success
-import scala.util.Try
+import scala.collection.mutable
+import scala.util.{Failure, Success, Try}
 
 class AstCreationPass(cpg: Cpg, astGenRunnerResult: AstGenRunnerResult, config: Config, report: Report = new Report())(
   implicit withSchemaValidation: ValidationMode
-) extends ForkJoinParallelCpgPass[(String, String)](cpg) {
+) extends ForkJoinParallelCpgPassWithAccumulator[(String, String), mutable.HashSet[String]](cpg) {
 
   private val logger: Logger = LoggerFactory.getLogger(classOf[AstCreationPass])
 
-  private val global = new Global()
+  private var collectedTypes: Set[String] = Set.empty
 
-  def typesSeen(): List[String] = global.usedTypes.keys().asScala.filterNot(_ == Defines.Any).toList
+  def typesSeen(): Set[String] = collectedTypes
+
+  override def createAccumulator(): mutable.HashSet[String] = mutable.HashSet.empty[String]
+
+  override def mergeAccumulator(left: mutable.HashSet[String], right: mutable.HashSet[String]): Unit = left ++= right
+
+  override def onAccumulatorComplete(builder: DiffGraphBuilder, accumulator: mutable.HashSet[String]): Unit = {
+    collectedTypes = accumulator.toSet
+  }
 
   override def generateParts(): Array[(String, String)] = astGenRunnerResult.parsedFiles.toArray
 
@@ -47,15 +49,19 @@ class AstCreationPass(cpg: Cpg, astGenRunnerResult: AstGenRunnerResult, config:
     }
   }
 
-  override def runOnPart(diffGraph: DiffGraphBuilder, input: (String, String)): Unit = {
+  override def runOnPart(
+    diffGraph: DiffGraphBuilder,
+    input: (String, String),
+    usedTypes: mutable.HashSet[String]
+  ): Unit = {
     val (rootPath, jsonFilename) = input
     val parseResultMaybe         = BabelJsonParser.readFile(Paths.get(rootPath), Paths.get(jsonFilename))
     val ((gotCpg, filename), duration) = TimeUtils.time {
       parseResultMaybe match {
         case Success(parseResult) =>
           report.addReportInfo(parseResult.filename, parseResult.fileLoc, parsed = true)
           Try {
-            val localDiff = new AstCreator(config, global, parseResult).createAst()
+            val localDiff = new AstCreator(config, usedTypes, parseResult).createAst()
             diffGraph.absorb(localDiff)
           } match {
             case Failure(exception) =>

joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/passes/JavaScriptTypeNodePass.scala

@@ -9,7 +9,7 @@ import scala.collection.mutable
 
 object JavaScriptTypeNodePass {
 
-  def withRegisteredTypes(registeredTypes: List[String], cpg: Cpg): TypeNodePass = {
+  def withRegisteredTypes(registeredTypes: Set[String], cpg: Cpg): TypeNodePass = {
     new TypeNodePass(registeredTypes, cpg, getTypesFromCpg = false) {
 
       override def fullToShortName(typeName: String): String = {

joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/Kotlin2Cpg.scala

@@ -245,7 +245,7 @@ class Kotlin2Cpg extends X2CpgFrontend with UsesService {
 
       val kotlinAstCreatorTypes = astCreator.usedTypes()
       val javaAstCreatorTypes   = runJavaSrcInterop(cpg, config, filesWithJavaExtension, kotlinAstCreatorTypes)
-      val allAstCreatorTypes    = (kotlinAstCreatorTypes.toSet ++ javaAstCreatorTypes.toSet).toList
+      val allAstCreatorTypes    = kotlinAstCreatorTypes.toSet ++ javaAstCreatorTypes.toSet
 
       TypeNodePass.withRegisteredTypes(allAstCreatorTypes, cpg).createAndApply()