[x2cpg] Update diffgraph and cfgcreator with support for new edges (#5890)

* Edge-first ControlStructureTraversal with fallback

* CfgCreator edge-first with order fallback

* Update CPG Version Number

* Incorporate review feedback

* Add new edges support to diffgraph

* Update Creator Base to auto-emit some edges where info is already provided

Author: SuperUserDone

build.sbt

@@ -2,7 +2,7 @@ name                     := "joern"
 ThisBuild / organization := "io.joern"
 ThisBuild / scalaVersion := "3.6.4"
 
-val cpgVersion = "1.7.60"
+val cpgVersion = "1.7.61"
 
 lazy val joerncli          = Projects.joerncli
 lazy val querydb           = Projects.querydb

joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/Ast.scala

@@ -32,6 +32,33 @@ object Ast {
     ast.conditionEdges.foreach { edge =>
       diffGraph.addEdge(edge.src, edge.dst, EdgeTypes.CONDITION)
     }
+    ast.trueBodyEdges.foreach { edge =>
+      diffGraph.addEdge(edge.src, edge.dst, EdgeTypes.TRUE_BODY)
+    }
+    ast.falseBodyEdges.foreach { edge =>
+      diffGraph.addEdge(edge.src, edge.dst, EdgeTypes.FALSE_BODY)
+    }
+    ast.doBodyEdges.foreach { edge =>
+      diffGraph.addEdge(edge.src, edge.dst, EdgeTypes.DO_BODY)
+    }
+    ast.tryBodyEdges.foreach { edge =>
+      diffGraph.addEdge(edge.src, edge.dst, EdgeTypes.TRY_BODY)
+    }
+    ast.catchBodyEdges.foreach { edge =>
+      diffGraph.addEdge(edge.src, edge.dst, EdgeTypes.CATCH_BODY)
+    }
+    ast.finallyBodyEdges.foreach { edge =>
+      diffGraph.addEdge(edge.src, edge.dst, EdgeTypes.FINALLY_BODY)
+    }
+    ast.forInitEdges.foreach { edge =>
+      diffGraph.addEdge(edge.src, edge.dst, EdgeTypes.FOR_INIT)
+    }
+    ast.forUpdateEdges.foreach { edge =>
+      diffGraph.addEdge(edge.src, edge.dst, EdgeTypes.FOR_UPDATE)
+    }
+    ast.forBodyEdges.foreach { edge =>
+      diffGraph.addEdge(edge.src, edge.dst, EdgeTypes.FOR_BODY)
+    }
     ast.receiverEdges.foreach { edge =>
       diffGraph.addEdge(edge.src, edge.dst, EdgeTypes.RECEIVER)
     }
@@ -90,6 +117,15 @@ case class Ast(
   ], // technically this should be a Seq[AstNewNode], but we also use it for non-ast nodes like Binding...
   edges: collection.Seq[AstEdge] = Vector.empty,
   conditionEdges: collection.Seq[AstEdge] = Vector.empty,
+  trueBodyEdges: collection.Seq[AstEdge] = Vector.empty,
+  falseBodyEdges: collection.Seq[AstEdge] = Vector.empty,
+  doBodyEdges: collection.Seq[AstEdge] = Vector.empty,
+  tryBodyEdges: collection.Seq[AstEdge] = Vector.empty,
+  catchBodyEdges: collection.Seq[AstEdge] = Vector.empty,
+  finallyBodyEdges: collection.Seq[AstEdge] = Vector.empty,
+  forInitEdges: collection.Seq[AstEdge] = Vector.empty,
+  forUpdateEdges: collection.Seq[AstEdge] = Vector.empty,
+  forBodyEdges: collection.Seq[AstEdge] = Vector.empty,
   refEdges: collection.Seq[AstEdge] = Vector.empty,
   bindsEdges: collection.Seq[AstEdge] = Vector.empty,
   receiverEdges: collection.Seq[AstEdge] = Vector.empty,
@@ -113,6 +149,15 @@ case class Ast(
         }
       ),
       conditionEdges = conditionEdges ++ other.conditionEdges,
+      trueBodyEdges = trueBodyEdges ++ other.trueBodyEdges,
+      falseBodyEdges = falseBodyEdges ++ other.falseBodyEdges,
+      doBodyEdges = doBodyEdges ++ other.doBodyEdges,
+      tryBodyEdges = tryBodyEdges ++ other.tryBodyEdges,
+      catchBodyEdges = catchBodyEdges ++ other.catchBodyEdges,
+      finallyBodyEdges = finallyBodyEdges ++ other.finallyBodyEdges,
+      forInitEdges = forInitEdges ++ other.forInitEdges,
+      forUpdateEdges = forUpdateEdges ++ other.forUpdateEdges,
+      forBodyEdges = forBodyEdges ++ other.forBodyEdges,
       argEdges = argEdges ++ other.argEdges,
       receiverEdges = receiverEdges ++ other.receiverEdges,
       refEdges = refEdges ++ other.refEdges,
@@ -126,6 +171,15 @@ case class Ast(
       nodes ++ other.nodes,
       edges = edges ++ other.edges,
       conditionEdges = conditionEdges ++ other.conditionEdges,
+      trueBodyEdges = trueBodyEdges ++ other.trueBodyEdges,
+      falseBodyEdges = falseBodyEdges ++ other.falseBodyEdges,
+      doBodyEdges = doBodyEdges ++ other.doBodyEdges,
+      tryBodyEdges = tryBodyEdges ++ other.tryBodyEdges,
+      catchBodyEdges = catchBodyEdges ++ other.catchBodyEdges,
+      finallyBodyEdges = finallyBodyEdges ++ other.finallyBodyEdges,
+      forInitEdges = forInitEdges ++ other.forInitEdges,
+      forUpdateEdges = forUpdateEdges ++ other.forUpdateEdges,
+      forBodyEdges = forBodyEdges ++ other.forBodyEdges,
       argEdges = argEdges ++ other.argEdges,
       receiverEdges = receiverEdges ++ other.receiverEdges,
       refEdges = refEdges ++ other.refEdges,
@@ -154,6 +208,56 @@ case class Ast(
     this.copy(conditionEdges = conditionEdges ++ List(AstEdge(src, dst)))
   }
 
+  def withTrueBodyEdge(src: NewNode, dst: NewNode): Ast = {
+    Ast.neighbourValidation(src, dst, EdgeTypes.TRUE_BODY)
+    this.copy(trueBodyEdges = trueBodyEdges ++ List(AstEdge(src, dst)))
+  }
+
+  def withFalseBodyEdge(src: NewNode, dst: NewNode): Ast = {
+    Ast.neighbourValidation(src, dst, EdgeTypes.FALSE_BODY)
+    this.copy(falseBodyEdges = falseBodyEdges ++ List(AstEdge(src, dst)))
+  }
+
+  def withDoBodyEdge(src: NewNode, dst: NewNode): Ast = {
+    Ast.neighbourValidation(src, dst, EdgeTypes.DO_BODY)
+    this.copy(doBodyEdges = doBodyEdges ++ List(AstEdge(src, dst)))
+  }
+
+  def withTryBodyEdge(src: NewNode, dst: NewNode): Ast = {
+    Ast.neighbourValidation(src, dst, EdgeTypes.TRY_BODY)
+    this.copy(tryBodyEdges = tryBodyEdges ++ List(AstEdge(src, dst)))
+  }
+
+  def withCatchBodyEdge(src: NewNode, dst: NewNode): Ast = {
+    Ast.neighbourValidation(src, dst, EdgeTypes.CATCH_BODY)
+    this.copy(catchBodyEdges = catchBodyEdges ++ List(AstEdge(src, dst)))
+  }
+
+  def withCatchBodyEdges(src: NewNode, dsts: List[NewNode]): Ast = {
+    dsts.foreach(dst => Ast.neighbourValidation(src, dst, EdgeTypes.CATCH_BODY))
+    this.copy(catchBodyEdges = catchBodyEdges ++ dsts.map(AstEdge(src, _)))
+  }
+
+  def withFinallyBodyEdge(src: NewNode, dst: NewNode): Ast = {
+    Ast.neighbourValidation(src, dst, EdgeTypes.FINALLY_BODY)
+    this.copy(finallyBodyEdges = finallyBodyEdges ++ List(AstEdge(src, dst)))
+  }
+
+  def withForInitEdge(src: NewNode, dst: NewNode): Ast = {
+    Ast.neighbourValidation(src, dst, EdgeTypes.FOR_INIT)
+    this.copy(forInitEdges = forInitEdges ++ List(AstEdge(src, dst)))
+  }
+
+  def withForUpdateEdge(src: NewNode, dst: NewNode): Ast = {
+    Ast.neighbourValidation(src, dst, EdgeTypes.FOR_UPDATE)
+    this.copy(forUpdateEdges = forUpdateEdges ++ List(AstEdge(src, dst)))
+  }
+
+  def withForBodyEdge(src: NewNode, dst: NewNode): Ast = {
+    Ast.neighbourValidation(src, dst, EdgeTypes.FOR_BODY)
+    this.copy(forBodyEdges = forBodyEdges ++ List(AstEdge(src, dst)))
+  }
+
   def withRefEdge(src: NewNode, dst: NewNode): Ast = {
     Ast.neighbourValidation(src, dst, EdgeTypes.REF)
     this.copy(refEdges = refEdges ++ List(AstEdge(src, dst)))
@@ -259,6 +363,15 @@ case class Ast(
 
     val newArgEdges       = argEdges.filter(_.src == node).map(x => AstEdge(newNode, newIfExists(x.dst)))
     val newConditionEdges = conditionEdges.filter(_.src == node).map(x => AstEdge(newNode, newIfExists(x.dst)))
+    val newTrueBodyEdges  = trueBodyEdges.filter(_.src == node).map(x => AstEdge(newNode, newIfExists(x.dst)))
+    val newFalseBodyEdges = falseBodyEdges.filter(_.src == node).map(x => AstEdge(newNode, newIfExists(x.dst)))
+    val newDoBodyEdges    = doBodyEdges.filter(_.src == node).map(x => AstEdge(newNode, newIfExists(x.dst)))
+    val newTryBodyEdges   = tryBodyEdges.filter(_.src == node).map(x => AstEdge(newNode, newIfExists(x.dst)))
+    val newCatchBodyEdges = catchBodyEdges.filter(_.src == node).map(x => AstEdge(newNode, newIfExists(x.dst)))
+    val newFinallyEdges   = finallyBodyEdges.filter(_.src == node).map(x => AstEdge(newNode, newIfExists(x.dst)))
+    val newForInitEdges   = forInitEdges.filter(_.src == node).map(x => AstEdge(newNode, newIfExists(x.dst)))
+    val newForUpdateEdges = forUpdateEdges.filter(_.src == node).map(x => AstEdge(newNode, newIfExists(x.dst)))
+    val newForBodyEdges   = forBodyEdges.filter(_.src == node).map(x => AstEdge(newNode, newIfExists(x.dst)))
     val newRefEdges       = refEdges.filter(_.src == node).map(x => AstEdge(newNode, newIfExists(x.dst)))
     val newBindsEdges     = bindsEdges.filter(_.src == node).map(x => AstEdge(newNode, newIfExists(x.dst)))
     val newReceiverEdges  = receiverEdges.filter(_.src == node).map(x => AstEdge(newNode, newIfExists(x.dst)))
@@ -268,6 +381,15 @@ case class Ast(
       .copy(
         argEdges = newArgEdges,
         conditionEdges = newConditionEdges,
+        trueBodyEdges = newTrueBodyEdges,
+        falseBodyEdges = newFalseBodyEdges,
+        doBodyEdges = newDoBodyEdges,
+        tryBodyEdges = newTryBodyEdges,
+        catchBodyEdges = newCatchBodyEdges,
+        finallyBodyEdges = newFinallyEdges,
+        forInitEdges = newForInitEdges,
+        forUpdateEdges = newForUpdateEdges,
+        forBodyEdges = newForBodyEdges,
         refEdges = newRefEdges,
         bindsEdges = newBindsEdges,
         receiverEdges = newReceiverEdges,

joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/AstCreatorBase.scala

@@ -182,7 +182,11 @@ abstract class AstCreatorBase[Node, NodeProcessor](filename: String)(implicit wi
     if (code.isDefined) {
       doWhileNode = doWhileNode.code(code.get)
     }
-    controlStructureAst(doWhileNode, condition, body, placeConditionLast = true)
+    val astWithChildren = controlStructureAst(doWhileNode, condition, body, placeConditionLast = true)
+    body.headOption.flatMap(_.root) match {
+      case Some(doBodyRoot) => astWithChildren.withDoBodyEdge(doWhileNode, doBodyRoot)
+      case None             => astWithChildren
+    }
   }
 
   def forAst(
@@ -210,17 +214,35 @@ abstract class AstCreatorBase[Node, NodeProcessor](filename: String)(implicit wi
     updateAsts: Seq[Ast],
     bodyAsts: Seq[Ast]
   ): Ast = {
-    val lineNumber  = forNode.lineNumber
-    val numOfLocals = locals.size
+    val lineNumber     = forNode.lineNumber
+    val numOfLocals    = locals.size
+    val initBlock      = setOrderExplicitly(wrapMultipleInBlock(initAsts, lineNumber), numOfLocals + 1)
+    val conditionBlock = setOrderExplicitly(wrapMultipleInBlock(conditionAsts, lineNumber), numOfLocals + 2)
+    val updateBlock    = setOrderExplicitly(wrapMultipleInBlock(updateAsts, lineNumber), numOfLocals + 3)
     // for the expected orders see CfgCreator.cfgForForStatement
     if (bodyAsts.nonEmpty) setOrderExplicitly(bodyAsts.head, numOfLocals + 4)
-    Ast(forNode)
+    val astWithChildren = Ast(forNode)
       .withChildren(locals)
-      .withChild(setOrderExplicitly(wrapMultipleInBlock(initAsts, lineNumber), numOfLocals + 1))
-      .withChild(setOrderExplicitly(wrapMultipleInBlock(conditionAsts, lineNumber), numOfLocals + 2))
-      .withChild(setOrderExplicitly(wrapMultipleInBlock(updateAsts, lineNumber), numOfLocals + 3))
+      .withChild(initBlock)
+      .withChild(conditionBlock)
+      .withChild(updateBlock)
       .withChildren(bodyAsts)
       .withConditionEdges(forNode, conditionAsts.flatMap(_.root).toList)
+
+    val astWithForInit = initBlock.root match {
+      case Some(initRoot) => astWithChildren.withForInitEdge(forNode, initRoot)
+      case None           => astWithChildren
+    }
+
+    val astWithForUpdate = updateBlock.root match {
+      case Some(updateRoot) => astWithForInit.withForUpdateEdge(forNode, updateRoot)
+      case None             => astWithForInit
+    }
+
+    bodyAsts.headOption.flatMap(_.root) match {
+      case Some(bodyRoot) => astWithForUpdate.withForBodyEdge(forNode, bodyRoot)
+      case None           => astWithForUpdate
+    }
   }
 
   /** For the given try body, catch ASTs and finally AST, create a try-catch-finally AST with orders set correctly for
@@ -248,10 +270,22 @@ abstract class AstCreatorBase[Node, NodeProcessor](filename: String)(implicit wi
     */
   def tryCatchAst(tryNode: NewControlStructure, tryBodyAst: Ast, catchAsts: Seq[Ast], finallyAst: Option[Ast]): Ast = {
     setArgumentIndices(tryBodyAst +: (catchAsts ++ finallyAst.toSeq))
-    Ast(tryNode)
+    val astWithChildren = Ast(tryNode)
       .withChild(tryBodyAst)
       .withChildren(catchAsts)
       .withChildren(finallyAst.toSeq)
+
+    val astWithTryBody = tryBodyAst.root match {
+      case Some(tryBodyRoot) => astWithChildren.withTryBodyEdge(tryNode, tryBodyRoot)
+      case None              => astWithChildren
+    }
+
+    val astWithCatchBodies = astWithTryBody.withCatchBodyEdges(tryNode, catchAsts.flatMap(_.root).toList)
+
+    finallyAst.flatMap(_.root) match {
+      case Some(finallyRoot) => astWithCatchBodies.withFinallyBodyEdge(tryNode, finallyRoot)
+      case None              => astWithCatchBodies
+    }
   }
 
   /** For a given block node and statement ASTs, create an AST that represents the block. The main purpose of this

joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/controlflow/cfgcreation/CfgCreator.scala

@@ -185,7 +185,11 @@ class CfgCreator(entryNode: Method, diffGraph: DiffGraphBuilder) {
     }
 
   protected def cfgForThrowStatement(node: ControlStructure): Cfg = {
-    val throwExprCfg     = node.astChildren.find(_.order == 1).map(cfgFor).getOrElse(Cfg.empty)
+    val throwExprCfg = Iterator(node)
+      .coalesce(_._argumentOut.cast[AstNode], _.astChildren.order(1))
+      .headOption
+      .map(cfgFor)
+      .getOrElse(Cfg.empty)
     val concatedNatedCfg = throwExprCfg ++ Cfg(entryNode = Option(node))
     concatedNatedCfg.copy(edges = concatedNatedCfg.edges ++ singleEdge(node, exitNode))
   }
@@ -373,12 +377,24 @@ class CfgCreator(entryNode: Method, diffGraph: DiffGraphBuilder) {
     * for the loop and a fringe.
     */
   protected def cfgForForStatement(node: ControlStructure): Cfg = {
-    val children     = node.astChildren.l
-    val nLocals      = children.count(_.isLocal)
-    val initExprCfg  = children.find(_.order == nLocals + 1).map(cfgFor).getOrElse(Cfg.empty)
+    val children = node.astChildren.l
+    val nLocals  = children.count(_.isLocal)
+    val initExprCfg = Iterator(node)
+      .coalesce(_._forInitOut.cast[AstNode], _.astChildren.order(nLocals + 1))
+      .headOption
+      .map(cfgFor)
+      .getOrElse(Cfg.empty)
     val conditionCfg = children.find(_.order == nLocals + 2).map(cfgFor).getOrElse(Cfg.empty)
-    val loopExprCfg  = children.find(_.order == nLocals + 3).map(cfgFor).getOrElse(Cfg.empty)
-    val bodyCfg      = children.find(_.order == nLocals + 4).map(cfgFor).getOrElse(Cfg.empty)
+    val loopExprCfg = Iterator(node)
+      .coalesce(_._forUpdateOut.cast[AstNode], _.astChildren.order(nLocals + 3))
+      .headOption
+      .map(cfgFor)
+      .getOrElse(Cfg.empty)
+    val bodyCfg = Iterator(node)
+      .coalesce(_._forBodyOut.cast[AstNode], _.astChildren.order(nLocals + 4))
+      .headOption
+      .map(cfgFor)
+      .getOrElse(Cfg.empty)
 
     val innerCfg      = bodyCfg ++ loopExprCfg
     val loopEntryNode = conditionCfg.entryNode.orElse(innerCfg.entryNode)
@@ -409,7 +425,11 @@ class CfgCreator(entryNode: Method, diffGraph: DiffGraphBuilder) {
     * inner CFG as bodyCfg ++ conditionCfg and then connect edges according to the semantics of do-while.
     */
   protected def cfgForDoStatement(node: ControlStructure): Cfg = {
-    val bodyCfg      = node.astChildren.where(_.order(1)).headOption.map(cfgFor).getOrElse(Cfg.empty)
+    val bodyCfg = Iterator(node)
+      .coalesce(_._doBodyOut.cast[AstNode], _.astChildren.order(1))
+      .headOption
+      .map(cfgFor)
+      .getOrElse(Cfg.empty)
     val conditionCfg = node.condition.headOption.map(cfgFor).getOrElse(Cfg.empty)
     val innerCfg     = bodyCfg ++ conditionCfg
 
@@ -516,41 +536,42 @@ class CfgCreator(entryNode: Method, diffGraph: DiffGraphBuilder) {
     * field of the `Block` node must be set to `finally`.
     */
   protected def cfgForTryStatement(node: ControlStructure): Cfg = {
-    val maybeTryBlock =
-      node.astChildren
-        .order(1)
-        .where(_.astChildren) // Filter out empty `try` bodies
-        .headOption
+    val maybeTryBlock = Iterator(node)
+      .coalesce(
+        _._tryBodyOut.cast[AstNode].filter(_.astChildren.nonEmpty),
+        _.astChildren.order(1).where(_.astChildren) // Filter out empty `try` bodies
+      )
+      .headOption
 
     val tryBodyCfg: Cfg = maybeTryBlock.map(cfgFor).getOrElse(Cfg.empty)
 
     val catchControlStructures =
       (node.astChildren.isControlStructure.isCatch ++ node.astChildren.isControlStructure.isElse).toList
-    val catchBodyCfgs = if (catchControlStructures.isEmpty) {
-      node.astChildren.order(2).toList match {
-        case Nil  => List(Cfg.empty)
-        case asts => asts.map(cfgFor)
-      }
-    } else {
-      catchControlStructures match {
-        case Nil  => List(Cfg.empty)
-        case asts => asts.map(cfgFor)
-      }
+    val catchBodyFallback =
+      if (catchControlStructures.isEmpty) node.astChildren.order(2)
+      else catchControlStructures.iterator
+
+    val catchBodyCfgs = Iterator(node)
+      .coalesce(_._catchBodyOut.cast[AstNode], _ => catchBodyFallback)
+      .map(cfgFor)
+      .toList match {
+      case Nil  => List(Cfg.empty)
+      case asts => asts
     }
 
     val finallyControlStructures = node.astChildren.isControlStructure.isFinally.toList
-    val maybeFinallyBodyCfg = if (catchControlStructures.isEmpty && finallyControlStructures.isEmpty) {
-      node.astChildren
-        .order(3)
-        .map(cfgFor)
-        .headOption // Assume there can only be one
-        .toList
-    } else {
-      finallyControlStructures
-        .map(cfgFor)
-        .headOption // Assume there can only be one
-        .toList
-    }
+    val finallyBodyFallback =
+      if (catchControlStructures.isEmpty && finallyControlStructures.isEmpty) {
+        node.astChildren.order(3)
+      } else {
+        finallyControlStructures.iterator
+      }
+
+    val maybeFinallyBodyCfg = Iterator(node)
+      .coalesce(_._finallyBodyOut.cast[AstNode], _ => finallyBodyFallback)
+      .map(cfgFor)
+      .headOption // Assume there can only be one
+      .toList
 
     val tryToCatchEdges = catchBodyCfgs.flatMap { catchBodyCfg =>
       edgesFromFringeTo(tryBodyCfg, catchBodyCfg.entryNode)