Revert new pass-based approach to fix smaller problems in peace (#220)

* Revert "Fix `runWithPreprocessorAndOutput` (#219)"

This reverts commit 5826306.

* Revert "Commented AST to CFG converter - functional version (#217)"

This reverts commit 19fbee0.

* Revert "Include `JumpTarget` nodes in the CFG (#216)"

This reverts commit 9dc2098.

* Revert "Pass-Based FuzzyC2Cpg (#214)"

This reverts commit 3feb6c7.

Author: fabsx00

build.sbt

@@ -3,7 +3,7 @@ organization := "io.shiftleft"
 scalaVersion := "2.13.1"
 enablePlugins(GitVersioning)
 
-val cpgVersion = "0.11.338"
+val cpgVersion = "0.11.334"
 val antlrVersion = "4.7.2"
 
 libraryDependencies ++= Seq(

fuzzyc2cpg.sh

@@ -3,4 +3,4 @@
 SCRIPT_ABS_PATH=$(readlink -f "$0")
 SCRIPT_ABS_DIR=$(dirname $SCRIPT_ABS_PATH)
 
-$SCRIPT_ABS_DIR/target/universal/stage/bin/fuzzyc2cpg -J-XX:+UseG1GC -J-XX:CompressedClassSpaceSize=128m -J-XX:+UseStringDeduplication -Dlogback.configurationFile=$SCRIPT_ABS_DIR/config/logback.xml $@
+$SCRIPT_ABS_DIR/target/universal/stage/bin/fuzzyc2cpg -Dlogback.configurationFile=$SCRIPT_ABS_DIR/config/logback.xml $@

src/main/java/io/shiftleft/fuzzyc2cpg/ast/statements/jump/GotoStatement.java

@@ -12,10 +12,6 @@ public String getTargetName() {
     return getChild(0).getEscapedCodeStr();
   }
 
-  public String getEscapedCodeStr() {
-    return "goto " + getTargetName() + ";";
-  }
-
   public void accept(ASTNodeVisitor visitor) {
     visitor.visit(this);
   }

src/main/java/io/shiftleft/fuzzyc2cpg/output/inmemory/OutputModule.java

@@ -0,0 +1,47 @@
+package io.shiftleft.fuzzyc2cpg.output.inmemory;
+
+import io.shiftleft.codepropertygraph.Cpg;
+import io.shiftleft.codepropertygraph.cpgloading.ProtoCpgLoader;
+import io.shiftleft.fuzzyc2cpg.output.CpgOutputModule;
+import overflowdb.OdbConfig;
+import io.shiftleft.proto.cpg.Cpg.CpgStruct;
+
+import java.util.LinkedList;
+import java.util.List;
+
+public class OutputModule implements CpgOutputModule {
+
+  private LinkedList<CpgStruct.Builder> cpgBuilders;
+  private Cpg cpg;
+
+  protected OutputModule() {
+    this.cpgBuilders = new LinkedList<>();
+  }
+
+  public Cpg getInternalGraph() {
+    return cpg;
+  }
+
+  @Override
+  public void setOutputIdentifier(String identifier) { }
+
+  @Override
+  public void persistCpg(CpgStruct.Builder cpg) {
+    synchronized (cpgBuilders) {
+      cpgBuilders.add(cpg);
+    }
+  }
+
+  public void persist() {
+    CpgStruct.Builder mergedBuilder = CpgStruct.newBuilder();
+
+    cpgBuilders.forEach(builder -> {
+      mergedBuilder.mergeFrom(builder.build());
+    });
+
+    List<CpgStruct> list = new LinkedList<>();
+    list.add(mergedBuilder.build());
+    cpg = ProtoCpgLoader.loadFromListOfProtos(list, OdbConfig.withoutOverflow());
+  }
+
+}

src/main/java/io/shiftleft/fuzzyc2cpg/output/inmemory/OutputModuleFactory.java

@@ -0,0 +1,37 @@
+package io.shiftleft.fuzzyc2cpg.output.inmemory;
+
+import io.shiftleft.codepropertygraph.Cpg;
+import io.shiftleft.fuzzyc2cpg.output.CpgOutputModule;
+import io.shiftleft.fuzzyc2cpg.output.CpgOutputModuleFactory;
+
+public class OutputModuleFactory implements CpgOutputModuleFactory {
+
+  private OutputModule outputModule;
+
+  @Override
+  public CpgOutputModule create() {
+    synchronized (this) {
+      if (outputModule == null) {
+        outputModule = new OutputModule();
+      }
+    }
+    return outputModule;
+  }
+
+  /**
+   * An internal representation of the graph.
+   *
+   * @return the internally constructed graph
+   */
+  public Cpg getInternalGraph() {
+    return outputModule.getInternalGraph();
+  }
+
+  @Override
+  public void persist() {
+    if (outputModule != null) {
+      outputModule.persist();
+    }
+  }
+}
+

src/main/java/io/shiftleft/fuzzyc2cpg/output/protobuf/OutputModule.java

@@ -0,0 +1,98 @@
+package io.shiftleft.fuzzyc2cpg.output.protobuf;
+
+import com.google.common.hash.HashFunction;
+import com.google.common.hash.Hasher;
+import com.google.common.hash.Hashing;
+import io.shiftleft.fuzzyc2cpg.output.CpgOutputModule;
+import io.shiftleft.proto.cpg.Cpg.CpgStruct;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.concurrent.ThreadLocalRandom;
+
+public class OutputModule implements CpgOutputModule {
+  private final Logger logger = LoggerFactory.getLogger(getClass());
+
+  private final Path protoTempDir;
+  private final boolean writeToDisk;
+
+  private String outputIdentifier;
+
+  public OutputModule(boolean writeToDisk,
+                      Path protoTempDir) {
+    this.writeToDisk = writeToDisk;
+    this.protoTempDir = protoTempDir;
+  }
+
+  @Override
+  public void setOutputIdentifier(String identifier) {
+    outputIdentifier = identifier;
+  }
+
+  /**
+   * This is called for each code property graph. There is one
+   * code property graph per method, and one graph for the overall
+   * program structure.
+   * */
+
+  @Override
+  public void persistCpg(CpgStruct.Builder cpg) throws IOException {
+    CpgStruct buildCpg = cpg.build();
+    if (writeToDisk) {
+      String outputFilename = getOutputFileName();
+      try (FileOutputStream outStream = new FileOutputStream(outputFilename)) {
+        buildCpg.writeTo(outStream);
+      }
+    }
+  }
+
+  /**
+   * The complete handling for an already existing file should not be necessary.
+   * This was added as a last resort to not get incomplete cpgs.
+   * In case we have a hash collision, the resulting cpg part file names will not
+   * be identical over different java2cpg runs.
+   */
+  private String getOutputFileName() {
+    String outputFilename = null;
+    int postfix = 0;
+    boolean fileExists = true;
+    int resolveAttemptCounter = 0;
+
+    while (fileExists && resolveAttemptCounter < 10) {
+      outputFilename = generateOutputFilename(postfix);
+      if (Files.exists(Paths.get(outputFilename))) {
+        postfix = ThreadLocalRandom.current().nextInt(0, 100000);
+
+        logger.warn("Hash collision identifier={}, postfix={}." +
+            " Retry with random postfix.", outputIdentifier, postfix);
+
+        resolveAttemptCounter++;
+      } else {
+        fileExists = false;
+      }
+    }
+
+    if (fileExists) {
+      logger.error("Unable to resolve hash collision. Cpg will be incomplete");
+    }
+
+    return outputFilename;
+  }
+
+  private String generateOutputFilename(int postfix) {
+    HashFunction hashFunction = Hashing.murmur3_128();
+
+    Hasher hasher = hashFunction.newHasher();
+    hasher.putUnencodedChars(outputIdentifier);
+    hasher.putInt(postfix);
+
+    String protoSuffix = ".bin";
+    return protoTempDir.toString() + File.separator + hasher.hash() + protoSuffix;
+  }
+}

src/main/java/io/shiftleft/fuzzyc2cpg/output/protobuf/OutputModuleFactory.java

@@ -0,0 +1,62 @@
+package io.shiftleft.fuzzyc2cpg.output.protobuf;
+
+import io.shiftleft.fuzzyc2cpg.output.CpgOutputModule;
+import io.shiftleft.fuzzyc2cpg.output.CpgOutputModuleFactory;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.io.FileUtils;
+
+public class OutputModuleFactory implements CpgOutputModuleFactory {
+
+  private final List<OutputModule> outputModules = new ArrayList<>();
+  private final boolean writeToDisk;
+  private final Path protoTempDir;
+  private final String outputFilename;
+
+  public OutputModuleFactory(String outputFilename,
+                             boolean writeToDisk) throws IOException {
+    this.writeToDisk = writeToDisk;
+    this.protoTempDir = Files.createTempDirectory("proto");
+    this.outputFilename = outputFilename;
+  }
+
+  @Override
+  public CpgOutputModule create() {
+    OutputModule outputModule = new OutputModule(writeToDisk, protoTempDir);
+    synchronized (this) {
+      outputModules.add(outputModule);
+    }
+    return outputModule;
+  }
+
+  /**
+   * Store collected CPGs into the output directory specified
+   * for this output module.
+   * Note: This method should be called only once all intermediate CPGs
+   * have been processed and collected.
+   * If the output module was configured to combine intermediate CPGs into a single
+   * one, we will combine individual proto files.
+   * */
+  @Override
+  public void persist() throws IOException {
+    if (writeToDisk) {
+      try {
+        ThreadedZipper threadedZipper = new ThreadedZipper(protoTempDir, outputFilename);
+        threadedZipper.start();
+        // wait until the thread is finished
+        // if we don't wait, the output folder
+        // may be deleted and and we get null pointer
+        threadedZipper.join();
+      } catch (InterruptedException interruptedException) {
+        throw new IOException(interruptedException);
+      }
+    }
+    if (this.protoTempDir != null && Files.exists(this.protoTempDir)) {
+      FileUtils.deleteDirectory(this.protoTempDir.toFile());
+    }
+  }
+}