update to 1.4.3 and --application-regex parameter

Author: waderwu

README.md

@@ -13,12 +13,15 @@ generate facts from bytecode (source is https://github.com/plast-lab/doop-mirror
 
 ## usage
 
+1.4.3
+
 ```
 Usage: soot-fact-generator [options] file...
 Options:
   --main <class>                        Specify the name of the main class.
   --ssa                                 Generate SSA facts, enabling flow-sensitive analysis.
   --full                                Generate facts by full transitive resolution.
+  --applicaiton-regex                   Application class glob expr default is **
   --allow-phantom                       Allow phantom classes.
   -d <directory>                        Specify where to generate output fact files.
   -i <archive>                          Find classes in <archive>.
@@ -52,7 +55,7 @@ Supported input archive formats: AAR, APK, JAR, ZIP
 ```
 常见的用法是
 ```
-java -jar soot-fact-generator.jar -i input.jar  -l /usr/lib/jvm/java-8-oracle/jre/lib/rt.jar --generate-jimple --allow-phantom --full -d out
+java -jar soot-fact-generator-1.4.3.jar -i Benchmark.jar  -l /usr/lib/jvm/java-8-oracle/jre/lib/rt.jar --generate-jimple --allow-phantom --full --ignore-factgen-errors  --ignore-wrong-staticness  --application-regex 'com.bytecodedl.benchmark.**' -d out 
 ```
 其中
 - `-i` 指定待分析的jar包
@@ -61,6 +64,9 @@ java -jar soot-fact-generator.jar -i input.jar  -l /usr/lib/jvm/java-8-oracle/jr
 - `--allow-phantom` 大概是允许解析依赖不存在的类
 - `--full` 表示对所有class进行解析
 - `-d` 指定输出目录
+- `--ignore-factgen-errors  --ignore-wrong-staticness` 忽略生成fact过程中的一些错误
+- `--application-regex 'com.bytecodedl.benchmark.**'` 表示只对`com.bytecodedl.benchmark.`开头的class解析method body，其他只解析函数签名
+  - 如果存在多个开头，可以用`/`隔开，默认是`**`全部解析，如果只关注某些class中的逻辑建议加上该参数，能极大降低生成fact的时间
 
 另外还额外增加了
 - `-i-dir` 指定待分析的jar目录

gradle.properties

@@ -1 +1 @@
-version=1.4.2
+version=1.4.3

src/main/java/org/clyze/doop/common/BasicJavaSupport.java

@@ -46,9 +46,10 @@ public ExecutorService getExecutor() {
      * Helper method to read classes and resources from input archives.
      */
     public void preprocessInputs(Database db, Set<String> tmpDirs) throws IOException {
+        Set<String> tmpClasses = ConcurrentHashMap.newKeySet();
         for (String filename : parameters.getInputs()) {
             logger.info("Preprocessing application: " + filename);
-            preprocessInput(db, tmpDirs, classesInApplicationJars, filename);
+            preprocessInput(db, tmpDirs, tmpClasses, filename);
         }
         for (String filename : parameters.getPlatformLibs()) {
             logger.info("Preprocessing platform library: " + filename);
@@ -58,6 +59,18 @@ public void preprocessInputs(Database db, Set<String> tmpDirs) throws IOExceptio
             logger.info("Preprocessing dependency: " + filename);
             preprocessInput(db, tmpDirs, classesInDependencyJars, filename);
         }
+
+        classifyClasses(tmpClasses);
+    }
+
+    public void classifyClasses(Set<String> tmpClasses){
+        for (String filename : tmpClasses) {
+            if (parameters.isApplicationClass(filename)){
+                classesInApplicationJars.add(filename);
+            }else{
+                classesInLibraryJars.add(filename);
+            }
+        }
     }
 
     /**

src/main/java/org/clyze/doop/common/Parameters.java

@@ -360,7 +360,7 @@ public boolean isSpringBootJar(String jarFile){
             ZipEntry entry = null;
 
             while((entry = in.getNextEntry()) != null){
-                if (entry.isDirectory() && entry.getName().equals("org/springframework/boot/")){
+                if (entry.isDirectory() && entry.getName().equals("BOOT-INF/")){
                     return true;
                 }
             }

src/main/java/org/clyze/doop/soot/BasicJavaSupport_Soot.java

@@ -2,19 +2,27 @@
 
 import java.util.Collection;
 import java.util.Set;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.clyze.doop.common.ArtifactScanner;
 import org.clyze.doop.common.BasicJavaSupport;
 import soot.Scene;
 import soot.SootClass;
 import soot.SourceLocator;
 
 public class BasicJavaSupport_Soot extends BasicJavaSupport implements ClassAdder {
+    Logger logger = LogManager.getLogger(BasicJavaSupport_Soot.class);
 
     public BasicJavaSupport_Soot(SootParameters parameters, ArtifactScanner artScanner) {
         super(parameters, artScanner);
     }
 
     public void addSootClasses(Iterable<String> classesToLoad, Collection<SootClass> loadedClasses, Scene scene) {
+        logger.info("start addSootClasses");
+        if (classesToLoad instanceof Set){
+            logger.info("addSootClasses size " + ((Set<String>) classesToLoad).size());
+        }
         for (String className : classesToLoad) {
             if (className.contains("]") || className.contains("[") || className.contains(";")) {
                 System.err.println("WARNING: class name '" + className + "' is not supported, class will not be loaded.");
@@ -28,6 +36,7 @@ public void addSootClasses(Iterable<String> classesToLoad, Collection<SootClass>
                 throw ex;
             }
         }
+        logger.info("end addSootClasses loadedClasses" + loadedClasses.size());
     }
 
     @Override
@@ -49,7 +58,7 @@ public boolean isAppOrDepClass(String t) {
     public void addAppClasses(Set<SootClass> classes, Scene scene) {
         addSootClasses(classesInApplicationJars, classes, scene);
         addBasicClasses(scene);
-        System.out.println("Classes in input (application) jar(s): " + classesInApplicationJars.size());
+        logger.info("Classes in input (application) jar(s): " + classesInApplicationJars.size());
     }
 
     @Override

src/main/java/org/clyze/doop/soot/ClassHeapFinder.java

@@ -35,8 +35,13 @@ private void scan(Iterable<SootClass> classes) {
 
     private void scan(SootMethod m) {
         if (!m.hasActiveBody()) {
-            m.retrieveActiveBody();
             System.err.println("Preprocessing: found method without active body: " + m.getSignature());
+            try{
+                m.retrieveActiveBody();
+            }catch (Exception e){
+                e.printStackTrace();
+                return;
+            }
         }
         for (Unit u : m.getActiveBody().getUnits())
             if (u instanceof AssignStmt) {

src/main/java/org/clyze/doop/soot/FactGenerator.java

@@ -236,7 +236,7 @@ void generate(SootMethod m, SessionCounter session) {
         for(SootClass clazz: m.getExceptions())
             _writer.writeMethodDeclaresException(m, clazz);
 
-        if(!(m.isAbstract() || m.isNative())) {
+        if(!(m.isAbstract() || m.isNative()) && this.sootParameters.isApplicationClass(m.getDeclaringClass())) {
             if(!m.hasActiveBody()) {
                 // This instruction is the bottleneck of
                 // soot-fact-generation.

src/main/java/org/clyze/doop/soot/Main.java

@@ -8,6 +8,7 @@
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Predicate;
+import java.util.stream.Collectors;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -292,7 +293,8 @@ private static void invokeSoot(SootParameters sootParameters, Database db, Set<S
 
             if (!sootParameters._lowMem){
                 logger.info("Checking class heaps for missing types...");
-                Collection<String> unrecorded = new ClassHeapFinder().getUnrecordedTypes(classes);
+                Set<SootClass> appClasses = classes.stream().filter(sootParameters::isApplicationClass).collect(Collectors.toSet());
+                Collection<String> unrecorded = new ClassHeapFinder().getUnrecordedTypes(appClasses);
                 if (unrecorded.size() > 0) {
                     // If option is set, fail and notify caller that fact generation
                     // must run again with these classes added.

src/main/java/org/clyze/doop/soot/SootParameters.java

@@ -87,6 +87,7 @@ static void showHelp() {
         System.err.println("  --main <class>                        Specify the name of the main class.");
         System.err.println("  --ssa                                 Generate SSA facts, enabling flow-sensitive analysis.");
         System.err.println("  --full                                Generate facts by full transitive resolution.");
+        System.err.println("  --applicaiton-regex                   Application class glob expr default is **");
         System.err.println("  --allow-phantom                       Allow phantom classes.");
         System.err.println("  -d <directory>                        Specify where to generate output fact files.");
         System.err.println("  -i <archive>                          Find classes in <archive>.");