Skip to content
This repository was archived by the owner on Oct 15, 2020. It is now read-only.

Commit 532c196

Browse files
mpollmeiermpollmeier
authored
use odb traversal instead of tinkerpop api (#227)
* port to odb api (1/?) * fix tests * use odb traversal api instead of tinkerpop * port to latest odb without tp3 dependency * latest cpg * use custom public release * manual bintray release * back to automatic releases, use latest official cpg * annoying foo/woo gitignore
1 parent 1ec4305 commit 532c196

16 files changed

+123
-107
lines changed

‎.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@
55
/project/project/
66
/project/target/
77
/target
8+
/foo.c
9+
/woo.c
810

911
private-key.pem
1012

‎build.sbt‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ organization := "io.shiftleft"
33
scalaVersion := "2.13.1"
44
enablePlugins(GitVersioning)
55

6-
val cpgVersion = "0.11.406"
6+
val cpgVersion = "0.12.1"
77
val antlrVersion = "4.7.2"
88

99
libraryDependencies ++= Seq(
@@ -120,6 +120,6 @@ developers := List(
120120
url("https://github.com/julianthome")
121121
)
122122
)
123-
publishTo := sonatypePublishToBundle.value
123+
// publishTo := sonatypePublishToBundle.value
124124
Global / useGpg := false
125125
Global / onChangedBuildSource := ReloadOnSourceChanges

‎src/main/scala/io/shiftleft/fuzzyc2cpg/FuzzyC2Cpg.scala‎

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,7 @@ import io.shiftleft.codepropertygraph.Cpg
99
import io.shiftleft.fuzzyc2cpg.passes.{AstCreationPass, CMetaDataPass, CfgCreationPass, StubRemovalPass, TypeNodePass}
1010
import io.shiftleft.passes.IntervalKeyPool
1111
import io.shiftleft.x2cpg.SourceFiles
12-
import overflowdb.{OdbConfig, OdbGraph}
13-
12+
import overflowdb.{Config, Graph}
1413
import scala.collection.mutable.ListBuffer
1514
import scala.util.control.NonFatal
1615
import scala.jdk.CollectionConverters._
@@ -97,15 +96,15 @@ class FuzzyC2Cpg() {
9796
logger.info("Output file exists, removing: " + outputPath)
9897
outFile.delete()
9998
}
100-
OdbConfig.withDefaults.withStorageLocation(outputPath)
99+
Config.withDefaults.withStorageLocation(outputPath)
101100
}
102101
.getOrElse {
103-
OdbConfig.withDefaults()
102+
Config.withDefaults()
104103
}
105104

106-
val graph = OdbGraph.open(odbConfig,
107-
io.shiftleft.codepropertygraph.generated.nodes.Factories.allAsJava,
108-
io.shiftleft.codepropertygraph.generated.edges.Factories.allAsJava)
105+
val graph = Graph.open(odbConfig,
106+
io.shiftleft.codepropertygraph.generated.nodes.Factories.allAsJava,
107+
io.shiftleft.codepropertygraph.generated.edges.Factories.allAsJava)
109108
new Cpg(graph)
110109
}
111110

‎src/main/scala/io/shiftleft/fuzzyc2cpg/passes/cfgcreation/CfgCreator.scala‎

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import io.shiftleft.codepropertygraph.generated.nodes.CfgNode
55
import io.shiftleft.fuzzyc2cpg.passes.cfgcreation.Cfg.CfgEdgeType
66
import io.shiftleft.passes.DiffGraph
77
import io.shiftleft.semanticcpg.language._
8+
import overflowdb.traversal.Traversal
89

910
/**
1011
* Translation of abstract syntax trees into control flow graphs
@@ -323,8 +324,8 @@ class CfgCreator(entryNode: nodes.Method) {
323324
* edges according to the semantics of do-while.
324325
* */
325326
private def cfgForDoStatement(node: nodes.ControlStructure): Cfg = {
326-
val bodyCfg = node.astChildren.filter(_.order(1)).headOption.map(cfgFor).getOrElse(Cfg.empty)
327-
val conditionCfg = node.start.condition.headOption.map(cfgFor).getOrElse(Cfg.empty)
327+
val bodyCfg = node.astChildren.where(_.order(1)).headOption.map(cfgFor).getOrElse(Cfg.empty)
328+
val conditionCfg = Traversal.fromSingle(node).condition.headOption.map(cfgFor).getOrElse(Cfg.empty)
328329
val innerCfg = bodyCfg ++ conditionCfg
329330

330331
val diffGraphs =
@@ -346,8 +347,8 @@ class CfgCreator(entryNode: nodes.Method) {
346347
* where body is optional.
347348
* */
348349
private def cfgForWhileStatement(node: nodes.ControlStructure): Cfg = {
349-
val conditionCfg = node.start.condition.headOption.map(cfgFor).getOrElse(Cfg.empty)
350-
val trueCfg = node.start.whenTrue.headOption.map(cfgFor).getOrElse(Cfg.empty)
350+
val conditionCfg = Traversal.fromSingle(node).condition.headOption.map(cfgFor).getOrElse(Cfg.empty)
351+
val trueCfg = Traversal.fromSingle(node).whenTrue.headOption.map(cfgFor).getOrElse(Cfg.empty)
351352
val diffGraphs = edgesFromFringeTo(conditionCfg, trueCfg.entryNode) ++
352353
edgesFromFringeTo(trueCfg, conditionCfg.entryNode) ++
353354
edges(trueCfg.continues, conditionCfg.entryNode)
@@ -365,8 +366,8 @@ class CfgCreator(entryNode: nodes.Method) {
365366
* CFG creation for switch statements of the form `switch{ case $x: ... }`.
366367
* */
367368
private def cfgForSwitchStatement(node: nodes.ControlStructure): Cfg = {
368-
val conditionCfg = node.start.condition.headOption.map(cfgFor).getOrElse(Cfg.empty)
369-
val bodyCfg = node.start.whenTrue.headOption.map(cfgFor).getOrElse(Cfg.empty)
369+
val conditionCfg = Traversal.fromSingle(node).condition.headOption.map(cfgFor).getOrElse(Cfg.empty)
370+
val bodyCfg = Traversal.fromSingle(node).whenTrue.headOption.map(cfgFor).getOrElse(Cfg.empty)
370371
val diffGraphs = edgesToMultiple(conditionCfg.fringe.map(_._1), bodyCfg.caseLabels, CaseEdge)
371372

372373
val hasDefaultCase = bodyCfg.caseLabels.exists(x => x.asInstanceOf[nodes.JumpTarget].name == "default")
@@ -386,9 +387,9 @@ class CfgCreator(entryNode: nodes.Method) {
386387
* followed by `else body2`.
387388
* */
388389
private def cfgForIfStatement(node: nodes.ControlStructure): Cfg = {
389-
val conditionCfg = node.start.condition.headOption.map(cfgFor).getOrElse(Cfg.empty)
390-
val trueCfg = node.start.whenTrue.headOption.map(cfgFor).getOrElse(Cfg.empty)
391-
val falseCfg = node.start.whenFalse.headOption.map(cfgFor).getOrElse(Cfg.empty)
390+
val conditionCfg = Traversal.fromSingle(node).condition.headOption.map(cfgFor).getOrElse(Cfg.empty)
391+
val trueCfg = Traversal.fromSingle(node).whenTrue.headOption.map(cfgFor).getOrElse(Cfg.empty)
392+
val falseCfg = Traversal.fromSingle(node).whenFalse.headOption.map(cfgFor).getOrElse(Cfg.empty)
392393

393394
val diffGraphs = edgesFromFringeTo(conditionCfg, trueCfg.entryNode) ++
394395
edgesFromFringeTo(conditionCfg, falseCfg.entryNode)

‎src/test/scala/io/shiftleft/fuzzyc2cpg/CpgTestFixture.scala‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,11 @@
11
package io.shiftleft.fuzzyc2cpg
22

3-
import gremlin.scala.GraphAsScala
43
import io.shiftleft.codepropertygraph.Cpg
54
import io.shiftleft.fuzzyc2cpg.passes.{AstCreationPass, CMetaDataPass, CfgCreationPass, StubRemovalPass}
65
import io.shiftleft.passes.IntervalKeyPool
76
import io.shiftleft.semanticcpg.language._
87
import io.shiftleft.x2cpg.SourceFiles
8+
import overflowdb.traversal.TraversalSource
99

1010
case class CpgTestFixture(projectName: String) {
1111

@@ -22,6 +22,6 @@ case class CpgTestFixture(projectName: String) {
2222
}
2323
new StubRemovalPass(cpg).createAndApply()
2424

25-
def V = cpg.graph.asScala.V
25+
def traversalSource = TraversalSource(cpg.graph)
2626

2727
}

‎src/test/scala/io/shiftleft/fuzzyc2cpg/MethodCfgLayoutTests.scala‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,22 @@
11
package io.shiftleft.fuzzyc2cpg
22

3-
import gremlin.scala._
43
import io.shiftleft.codepropertygraph.generated.{EdgeTypes, NodeKeys, NodeTypes, Operators}
54
import org.scalatest.{Matchers, WordSpec}
5+
import overflowdb.traversal._
6+
import overflowdb.{Node, PropertyKey}
67

78
class MethodCfgLayoutTests extends WordSpec with Matchers with TraversalUtils {
89
val fixture = CpgTestFixture("methodcfglayout")
910

10-
implicit class VertexListWrapper(vertexList: List[Vertex]) {
11-
def expandCfg(): List[Vertex] = {
11+
implicit class VertexListWrapper(vertexList: List[Node]) {
12+
def expandCfg(): List[Node] = {
1213
vertexList.flatMap(_.start.out(EdgeTypes.CFG).l)
1314
}
1415

15-
def checkForSingleProperty(label: String, property: Key[String], value: String): Unit = {
16+
def checkForSingleProperty(label: String, propertyKey: PropertyKey[String], value: String): Unit = {
1617
vertexList.size shouldBe 1
1718
vertexList.head.label shouldBe label
18-
vertexList.head.value2(property) shouldBe value
19+
vertexList.head.property(propertyKey) shouldBe value
1920
}
2021

2122
def checkForSingle(label: String): Unit = {

‎src/test/scala/io/shiftleft/fuzzyc2cpg/MethodDeclTest.scala‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,20 @@
11
package io.shiftleft.fuzzyc2cpg
22

3-
import org.scalatest.{Matchers, WordSpec}
4-
53
import io.shiftleft.codepropertygraph.generated.{NodeKeys, NodeTypes}
4+
import org.scalatest.{Matchers, WordSpec}
65

76
class MethodDeclTest extends WordSpec with Matchers {
87

98
private val fixture = CpgTestFixture("methoddecl")
109

1110
"MethodDeclTest" should {
1211
"omit the method declaration in presence of a definition" in {
13-
val result = fixture.V
14-
.hasLabel(NodeTypes.METHOD)
12+
val result = fixture.traversalSource
13+
.label(NodeTypes.METHOD)
1514
.l
1615

1716
result.size shouldBe 1
18-
val signature = result.head.property[String](NodeKeys.SIGNATURE.name).value
17+
val signature = result.head.property(NodeKeys.SIGNATURE)
1918
signature shouldBe "int add (int,int)"
2019
}
2120
}

‎src/test/scala/io/shiftleft/fuzzyc2cpg/MethodHeaderTests.scala‎

Lines changed: 34 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -1,67 +1,70 @@
11
package io.shiftleft.fuzzyc2cpg
22

3-
import gremlin.scala._
4-
import io.shiftleft.codepropertygraph.generated.{EdgeTypes, EvaluationStrategies, NodeKeys, NodeTypes}
3+
import io.shiftleft.codepropertygraph.generated._
54
import org.scalatest.{Matchers, WordSpec}
5+
import overflowdb._
66

77
class MethodHeaderTests extends WordSpec with Matchers {
88
val fixture = CpgTestFixture("methodheader")
99

1010
"Method header" should {
1111

1212
"have correct METHOD node for method foo" in {
13-
val methods = fixture.V.hasLabel(NodeTypes.METHOD).has(NodeKeys.NAME -> "foo").l
13+
val methods = fixture.traversalSource.label(NodeTypes.METHOD).has(NodeKeys.NAME -> "foo").l
1414
methods.size shouldBe 1
15-
methods.head.value2(NodeKeys.IS_EXTERNAL) shouldBe false
16-
methods.head.value2(NodeKeys.FULL_NAME) shouldBe "foo"
17-
methods.head.value2(NodeKeys.SIGNATURE) shouldBe "int foo (int,int)"
18-
methods.head.value2(NodeKeys.LINE_NUMBER) shouldBe 1
19-
methods.head.value2(NodeKeys.COLUMN_NUMBER) shouldBe 0
20-
methods.head.value2(NodeKeys.LINE_NUMBER_END) shouldBe 3
21-
methods.head.value2(NodeKeys.COLUMN_NUMBER_END) shouldBe 0
22-
methods.head.value2(NodeKeys.CODE) shouldBe "foo (int x,int y)"
15+
val method = methods.head
16+
method.property(NodeKeys.IS_EXTERNAL) shouldBe false
17+
method.property(NodeKeys.FULL_NAME) shouldBe "foo"
18+
method.property(NodeKeys.SIGNATURE) shouldBe "int foo (int,int)"
19+
method.property(NodeKeys.LINE_NUMBER) shouldBe 1
20+
method.property(NodeKeys.COLUMN_NUMBER) shouldBe 0
21+
method.property(NodeKeys.LINE_NUMBER_END) shouldBe 3
22+
method.property(NodeKeys.COLUMN_NUMBER_END) shouldBe 0
23+
method.property(NodeKeys.CODE) shouldBe "foo (int x,int y)"
2324
}
2425

2526
"have correct METHOD_PARAMETER_IN nodes for method foo" in {
26-
val parameters = fixture.V
27-
.hasLabel(NodeTypes.METHOD)
27+
val parameters = fixture
28+
.traversalSource
29+
.label(NodeTypes.METHOD)
2830
.has(NodeKeys.NAME -> "foo")
2931
.out(EdgeTypes.AST)
3032
.hasLabel(NodeTypes.METHOD_PARAMETER_IN)
3133
.l
3234

3335
parameters.size shouldBe 2
34-
val param1Option = parameters.find(_.value2(NodeKeys.ORDER) == 1)
36+
val param1Option = parameters.find(_.property(NodeKeys.ORDER) == 1)
3537
param1Option.isDefined shouldBe true
36-
param1Option.get.value2(NodeKeys.CODE) shouldBe "int x"
37-
param1Option.get.value2(NodeKeys.NAME) shouldBe "x"
38-
param1Option.get.value2(NodeKeys.EVALUATION_STRATEGY) shouldBe EvaluationStrategies.BY_VALUE
39-
param1Option.get.value2(NodeKeys.LINE_NUMBER) shouldBe 1
40-
param1Option.get.value2(NodeKeys.COLUMN_NUMBER) shouldBe 8
38+
param1Option.get.property(NodeKeys.CODE) shouldBe "int x"
39+
param1Option.get.property(NodeKeys.NAME) shouldBe "x"
40+
param1Option.get.property(NodeKeys.EVALUATION_STRATEGY) shouldBe EvaluationStrategies.BY_VALUE
41+
param1Option.get.property(NodeKeys.LINE_NUMBER) shouldBe 1
42+
param1Option.get.property(NodeKeys.COLUMN_NUMBER) shouldBe 8
4143

42-
val param2Option = parameters.find(_.value2(NodeKeys.ORDER) == 2)
44+
val param2Option = parameters.find(_.property(NodeKeyNames.ORDER) == 2)
4345
param2Option.isDefined shouldBe true
4446
param2Option.isDefined shouldBe true
45-
param2Option.get.value2(NodeKeys.CODE) shouldBe "int y"
46-
param2Option.get.value2(NodeKeys.NAME) shouldBe "y"
47-
param2Option.get.value2(NodeKeys.EVALUATION_STRATEGY) shouldBe EvaluationStrategies.BY_VALUE
48-
param2Option.get.value2(NodeKeys.LINE_NUMBER) shouldBe 1
49-
param2Option.get.value2(NodeKeys.COLUMN_NUMBER) shouldBe 15
47+
param2Option.get.property(NodeKeys.CODE) shouldBe "int y"
48+
param2Option.get.property(NodeKeys.NAME) shouldBe "y"
49+
param2Option.get.property(NodeKeys.EVALUATION_STRATEGY) shouldBe EvaluationStrategies.BY_VALUE
50+
param2Option.get.property(NodeKeys.LINE_NUMBER) shouldBe 1
51+
param2Option.get.property(NodeKeys.COLUMN_NUMBER) shouldBe 15
5052
}
5153

5254
"have correct METHOD_RETURN node for method foo" in {
53-
val methodReturn = fixture.V
54-
.hasLabel(NodeTypes.METHOD)
55+
val methodReturn = fixture
56+
.traversalSource
57+
.label(NodeTypes.METHOD)
5558
.has(NodeKeys.NAME -> "foo")
5659
.out(EdgeTypes.AST)
5760
.hasLabel(NodeTypes.METHOD_RETURN)
5861
.l
5962

6063
methodReturn.size shouldBe 1
61-
methodReturn.head.value2(NodeKeys.CODE) shouldBe "RET"
62-
methodReturn.head.value2(NodeKeys.EVALUATION_STRATEGY) shouldBe EvaluationStrategies.BY_VALUE
63-
methodReturn.head.value2(NodeKeys.LINE_NUMBER) shouldBe 1
64-
methodReturn.head.value2(NodeKeys.COLUMN_NUMBER) shouldBe 0
64+
methodReturn.head.property(NodeKeys.CODE) shouldBe "RET"
65+
methodReturn.head.property(NodeKeys.EVALUATION_STRATEGY) shouldBe EvaluationStrategies.BY_VALUE
66+
methodReturn.head.property(NodeKeys.LINE_NUMBER) shouldBe 1
67+
methodReturn.head.property(NodeKeys.COLUMN_NUMBER) shouldBe 0
6568
}
6669

6770
}

‎src/test/scala/io/shiftleft/fuzzyc2cpg/MethodInternalLinkageTests.scala‎

Lines changed: 15 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,42 +1,44 @@
11
package io.shiftleft.fuzzyc2cpg
22

3-
import gremlin.scala._
43
import io.shiftleft.codepropertygraph.generated.{EdgeTypes, NodeKeys, NodeTypes}
54
import org.scalatest.{Matchers, WordSpec}
5+
import overflowdb._
6+
import overflowdb.traversal._
7+
68

79
class MethodInternalLinkageTests extends WordSpec with Matchers with TraversalUtils {
810
val fixture = CpgTestFixture("methodinternallinkage")
911

10-
implicit class VertexListWrapper(vertexList: List[Vertex]) {
11-
def expandAst(filterLabels: String*): List[Vertex] = {
12+
implicit class VertexListWrapper(vertexList: List[Node]) {
13+
def expandAst(filterLabels: String*): List[Node] = {
1214
if (filterLabels.nonEmpty) {
13-
vertexList.flatMap(_.start.out(EdgeTypes.AST).hasLabel(filterLabels.head, filterLabels.tail: _*).l)
15+
vertexList.flatMap(_.start.out(EdgeTypes.AST).hasLabel(filterLabels: _*).l)
1416
} else {
1517
vertexList.flatMap(_.start.out(EdgeTypes.AST).l)
1618
}
1719
}
1820

19-
def expandRef(): List[Vertex] = {
21+
def expandRef(): List[Node] = {
2022
vertexList.flatMap(_.start.out(EdgeTypes.REF).l)
2123
}
2224

23-
def filterOrder(order: Int): List[Vertex] = {
24-
vertexList.filter(_.valueOption(NodeKeys.ORDER).getOrElse(-1) == order)
25+
def filterOrder(order: Int): List[Node] = {
26+
vertexList.to(Traversal).has(NodeKeys.ORDER -> order).l
2527
}
2628

27-
def filterName(name: String): List[Vertex] = {
28-
vertexList.filter(_.valueOption(NodeKeys.NAME).getOrElse("") == name)
29+
def filterName(name: String): List[Node] = {
30+
vertexList.to(Traversal).has(NodeKeys.NAME -> name).l
2931
}
3032

31-
def checkForSingle[T](label: String, propertyName: Key[T], value: T): Unit = {
33+
def checkForSingle[T](label: String, propertyKey: PropertyKey[T], value: T): Unit = {
3234
vertexList.size shouldBe 1
3335
vertexList.head.label() shouldBe label
34-
vertexList.head.value2(propertyName) shouldBe value
36+
vertexList.head.property(propertyKey) shouldBe value
3537
}
3638

37-
def checkForSingle[T](propertyName: Key[T], value: T): Unit = {
39+
def checkForSingle[T](propertyKey: PropertyKey[T], value: T): Unit = {
3840
vertexList.size shouldBe 1
39-
vertexList.head.value2(propertyName) shouldBe value
41+
vertexList.head.property(propertyKey) shouldBe value
4042
}
4143
}
4244

0 commit comments

Comments
 (0)