update to 1.4.2 add logger and lowMem and exception handle

waderwu · waderwu · commit b990f7f00334 · 2024-01-06T15:24:39.000+08:00
diff --git a/README.md b/README.md
@@ -64,4 +64,10 @@ java -jar soot-fact-generator.jar -i input.jar  -l /usr/lib/jvm/java-8-oracle/jr
 
 另外还额外增加了
 - `-i-dir` 指定待分析的jar目录
-- `-ld-dir` 指定待分析的依赖目录
+- `-ld-dir` 指定待分析的依赖目录
+
+分析很大文件/多文件，比如weblogic这种，可以使用如下命令(需使用最新版)，可根据自己情况调整JVM的参数
+
+```
+java -Xms8g -Xmx12g -jar soot-fact-generator-1.4.2.jar -d weblogic-out -i-dir /root/ByteCodeDL/weblogic --lowMem --allow-phantom --full -lsystem --ignore-factgen-errors
+```
diff --git a/build.gradle b/build.gradle
@@ -10,7 +10,6 @@ gradle.rootProject.ext.commonsCliVersion='1.2'
 gradle.rootProject.ext.metadataVersion='2.4.1'
 gradle.rootProject.ext.groovyVersion='3.0.9'
 gradle.rootProject.ext.clueCommonVersion='3.25.3'
-gradle.rootProject.ext.log4jVersion='1.2.17'
 gradle.rootProject.ext.spockVersion='2.1-M2-groovy-3.0'
 gradle.rootProject.ext.asmVersion = '8.0.1'
 
@@ -38,8 +37,9 @@ dependencies {
     implementation 'net.dongliu:apk-parser:2.6.10'          // needed for reading binary XML entries from APK inputs
     implementation "org.antlr:antlr4-runtime:${antlrVersion}"
     api project(":native-scanner")
-    api "log4j:log4j:${log4jVersion}"            // Logging implementation
     implementation 'org.apache.commons:commons-collections4:4.1'
+    implementation 'org.apache.logging.log4j:log4j-api:2.17.1'
+    implementation 'org.apache.logging.log4j:log4j-core:2.17.1'
 
     testImplementation "org.codehaus.groovy:groovy-all:${groovyVersion}"  // Groovy
     testImplementation "org.spockframework:spock-core:${spockVersion}"
diff --git a/gradle.properties b/gradle.properties
@@ -1 +1 @@
-version=1.4.1
+version=1.4.2
diff --git a/src/main/java/org/clyze/doop/common/BasicJavaSupport.java b/src/main/java/org/clyze/doop/common/BasicJavaSupport.java
@@ -1,5 +1,8 @@
 package org.clyze.doop.common;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
@@ -24,6 +27,8 @@ public class BasicJavaSupport {
     private final ExecutorService exec = Executors.newFixedThreadPool(3);
     public final Collection<String> xmlRoots = ConcurrentHashMap.newKeySet();
 
+    private static final Logger logger = LogManager.getLogger(BasicJavaSupport.class);
+
     public BasicJavaSupport(Parameters parameters, ArtifactScanner artScanner) {
         this.parameters = parameters;
         this.artScanner = artScanner;
@@ -42,15 +47,15 @@ public ExecutorService getExecutor() {
      */
     public void preprocessInputs(Database db, Set<String> tmpDirs) throws IOException {
         for (String filename : parameters.getInputs()) {
-            System.out.println("Preprocessing application: " + filename);
+            logger.info("Preprocessing application: " + filename);
             preprocessInput(db, tmpDirs, classesInApplicationJars, filename);
         }
         for (String filename : parameters.getPlatformLibs()) {
-            System.out.println("Preprocessing platform library: " + filename);
+            logger.info("Preprocessing platform library: " + filename);
             preprocessInput(db, tmpDirs, classesInLibraryJars, filename);
         }
         for (String filename : parameters.getDependencies()) {
-            System.out.println("Preprocessing dependency: " + filename);
+            logger.info("Preprocessing dependency: " + filename);
             preprocessInput(db, tmpDirs, classesInDependencyJars, filename);
         }
     }
@@ -82,21 +87,21 @@ else if ((isJar || isAar || isZip || isWar) && entryName.endsWith(".xml")) {
                 // APK archives may contain binary XML and need decoding.
                 File xmlTmpFile = ArtifactScanner.extractZipEntryAsFile("xml-file", jarFile, entry, entryName);
                 if (parameters._debug)
-                    System.out.println("Processing XML entry (in " + filename + "): " + entryName);
+                    logger.info("Processing XML entry (in " + filename + "): " + entryName);
                 XMLFactGenerator.processFile(xmlTmpFile, db, "", parameters._debug);
             }
         };
 
         if (isWar) {
-            System.out.println("Processing WAR: " + filename);
+            logger.info("Processing WAR: " + filename);
             // Process WAR inputs.
             parameters.processFatArchives(tmpDirs);
         }else if (isSpringBoot) {
-            System.out.println("Processing springBoot: " + filename);
+            logger.info("Processing springBoot: " + filename);
             parameters.processSpringBootArchives(tmpDirs, filename);
             parameters.getInputs().forEach(file -> {
                 try{
-//                    System.out.println(file);
+//                    logger.info(file);
                     artScanner.processArchive(file, classSet::add, gProc);
                 }catch (Exception e){
                     e.printStackTrace();
@@ -112,7 +117,7 @@ else if ((isJar || isAar || isZip || isWar) && entryName.endsWith(".xml")) {
                 artScanner.processClass(fis, f, classSet::add);
             }
         } else
-            System.err.println("WARNING: artifact scanner skips " + filename);
+            logger.warn("WARNING: artifact scanner skips " + filename);
     }
 
     public PropertyProvider getPropertyProvider() {
diff --git a/src/main/java/org/clyze/doop/common/Driver.java b/src/main/java/org/clyze/doop/common/Driver.java
@@ -1,5 +1,8 @@
 package org.clyze.doop.common;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
 import java.util.Collections;
 import java.util.Set;
 import java.util.concurrent.*;
@@ -15,18 +18,20 @@ public abstract class Driver<C> {
     protected Set<C> _tmpClassGroup;
     private int _classCounter;
     private final int _totalClasses;
-    private final int _classSplit = 80;
+    private final int _classSplit = 800;
     private int errors;
     public final boolean _ignoreFactGenErrors;
 
+    private static final Logger logger = LogManager.getLogger(Driver.class);
+
     protected Driver(int totalClasses, Integer cores, boolean ignoreFactGenErrors) {
         this._totalClasses = totalClasses;
         this._cores = cores == null? Runtime.getRuntime().availableProcessors() : cores;
         this._classCounter = 0;
         initTmpClassGroup();
         this._ignoreFactGenErrors = ignoreFactGenErrors;
 
-        System.out.println("Fact generation cores: " + _cores);
+        logger.info("Fact generation cores: " + _cores);
     }
 
     protected void initTmpClassGroup() {
@@ -67,9 +72,9 @@ public static void waitForExecutorShutdown(ExecutorService executor) throws Doop
         try {
             boolean res = executor.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS);
             if (!res)
-                System.out.println("Executor timeout elapsed.");
+                logger.info("Executor timeout elapsed.");
         } catch (InterruptedException e) {
-            System.err.println(e.getMessage());
+           logger.error("waitForExecutorShutdown got interrupt", e);
             throw DoopErrorCodeException.error10();
         }
     }
@@ -78,7 +83,7 @@ private void shutdownExecutor() throws DoopErrorCodeException {
         waitForExecutorShutdown(_executor);
         if (errorsExist()) {
             String msg = "Fact generation failed (" + errors + " errors).";
-            System.err.println(msg);
+            logger.error(msg);
             if (!_ignoreFactGenErrors)
                 throw DoopErrorCodeException.error5(msg);
         }
@@ -98,6 +103,7 @@ private void generate(C curClass) {
         _tmpClassGroup.add(curClass);
 
         if ((_classCounter % _classSplit == 0) || (_classCounter == _totalClasses)) {
+            logger.info("have submit class {} for fact gen", _classCounter);
             _executor.execute(getFactGenRunnable());
             initTmpClassGroup();
         }
diff --git a/src/main/java/org/clyze/doop/soot/FactGenerator.java b/src/main/java/org/clyze/doop/soot/FactGenerator.java
@@ -7,6 +7,9 @@
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.clyze.doop.common.InstrInfo;
 import org.clyze.doop.common.Phantoms;
 import org.clyze.doop.common.SessionCounter;
@@ -34,6 +37,8 @@ class FactGenerator implements Runnable {
     private final SootParameters sootParameters;
     private final SootDriver _driver;
 
+    private static final Logger logger = LogManager.getLogger(FactGenerator.class);
+
     FactGenerator(FactWriter writer, Set<SootClass> sootClasses, SootDriver driver, SootParameters sootParameters, Phantoms phantoms)
     {
         this._writer = writer;
@@ -46,55 +51,87 @@ class FactGenerator implements Runnable {
 
     @Override
     public void run() {
-        final boolean ignoreErrors = _driver._ignoreFactGenErrors;
+        logger.info("start generate facts");
+        try{
+            final boolean ignoreErrors = _driver._ignoreFactGenErrors;
 
-        if (!ignoreErrors && _driver.errorsExist())
-            return;
+            if (!ignoreErrors && _driver.errorsExist())
+                return;
+
+            for (SootClass _sootClass : _sootClasses) {
+                try {
+                    _writer.writeClassOrInterfaceType(_sootClass);
+
+                    for (String mod : getModifiers(_sootClass.getModifiers(), false))
+                        if (!mod.trim().equals(""))
+                            _writer.writeClassModifier(_sootClass, mod);
 
-        for (SootClass _sootClass : _sootClasses) {
-            _writer.writeClassOrInterfaceType(_sootClass);
+                    // the isInterface condition prevents Object as superclass of interface
+                    if (_sootClass.hasSuperclass() && !_sootClass.isInterface()) {
+                        _writer.writeDirectSuperclass(_sootClass, _sootClass.getSuperclass());
+                    }
 
-            for (String mod : getModifiers(_sootClass.getModifiers(), false))
-                if (!mod.trim().equals(""))
-                    _writer.writeClassModifier(_sootClass, mod);
+                    for (SootClass i : _sootClass.getInterfaces()) {
+                        _writer.writeDirectSuperinterface(_sootClass, i);
+                    }
 
-            // the isInterface condition prevents Object as superclass of interface
-            if (_sootClass.hasSuperclass() && !_sootClass.isInterface()) {
-                _writer.writeDirectSuperclass(_sootClass, _sootClass.getSuperclass());
+                    _sootClass.getFields().forEach(this::generate);
+
+                    for (SootMethod m : new ArrayList<>(_sootClass.getMethods())) {
+                        SessionCounter session = new SessionCounter();
+                        try {
+                            generate(m, session);
+                        } catch (Throwable t) {
+                            // Map<Thread,StackTraceElement[]> liveThreads = Thread.getAllStackTraces();
+                            // for (Iterator<Thread> i = liveThreads.keySet().iterator(); i.hasNext(); ) {
+                            //     Thread key = i.next();
+                            //     logger.error("Thread " + key.getName());
+                            //     StackTraceElement[] trace = liveThreads.getLibrary(key);
+                            //     for (int j = 0; j < trace.length; j++) {
+                            //         logger.error("\tat " + trace[j]);
+                            //     }
+                            // }
+                            String msg = "Error while processing method: " + m ;
+                            logger.error(msg, t);
+                            if (!ignoreErrors) {
+                                // Inform the driver. This is safer than throwing an
+                                // exception, since it could be lost due to the executor
+                                // service running this class.
+                                _driver.markError();
+                                return;
+                            }
+                        }
+                    }
+                }catch (Exception e){
+                    if (_sootClass != null){
+                        logger.error("handle class {}.{} error",_sootClass.getJavaStyleName(), e);
+                    }
+                }
             }
 
-            for (SootClass i : _sootClass.getInterfaces()) {
-                _writer.writeDirectSuperinterface(_sootClass, i);
+            if (sootParameters._lowMem && !sootParameters._generateJimple){
+                gcMethodBody(_sootClasses);
             }
+        }catch (Exception e){
+            logger.error("FactGenerator run occur error ", e);
+        }
 
-            _sootClass.getFields().forEach(this::generate);
+        logger.info("finish generator facts");
 
-            for (SootMethod m : new ArrayList<>(_sootClass.getMethods())) {
-                SessionCounter session = new SessionCounter();
-                try {
-                    generate(m, session);
-                } catch (Throwable t) {
-                    // Map<Thread,StackTraceElement[]> liveThreads = Thread.getAllStackTraces();
-                    // for (Iterator<Thread> i = liveThreads.keySet().iterator(); i.hasNext(); ) {
-                    //     Thread key = i.next();
-                    //     System.err.println("Thread " + key.getName());
-                    //     StackTraceElement[] trace = liveThreads.getLibrary(key);
-                    //     for (int j = 0; j < trace.length; j++) {
-                    //         System.err.println("\tat " + trace[j]);
-                    //     }
-                    // }
-                    String msg = "Error while processing method: " + m + ": " + t.getMessage();
-                    System.err.println(msg);
-                    if (!ignoreErrors) {
-                        // Inform the driver. This is safer than throwing an
-                        // exception, since it could be lost due to the executor
-                        // service running this class.
-                        _driver.markError();
-                        return;
-                    }
+    }
+
+    private void gcMethodBody(Set<SootClass> sootClasses){
+        logger.info("start gcMethodBody");
+        for (SootClass sootClass : sootClasses) {
+            for (SootMethod method : sootClass.getMethods()) {
+                if (method.hasActiveBody()){
+                    method.setActiveBody(null);
                 }
             }
         }
+        logger.info("start System.gc");
+        System.gc();
+        logger.info("finish gcMethodBody");
     }
 
     private void generate(SootField f)
@@ -207,7 +244,7 @@ void generate(SootMethod m, SessionCounter session) {
                 m.retrieveActiveBody();
                 // } // synchronizing so broadly = giving up on Soot's races
 
-                // System.err.println("Found method without active body: " + m.getSignature());
+                // logger.error("Found method without active body: " + m.getSignature());
                 methodsWithoutActiveBodies.incrementAndGet();
             }
 
@@ -227,8 +264,7 @@ void generate(SootMethod m, SessionCounter session) {
                         m.setActiveBody(b0);
                 }
             } catch (RuntimeException ex) {
-                System.err.println("Fact generation failed for method " + m.getSignature() + ".");
-                ex.printStackTrace();
+                logger.error("Fact generation failed for method " + m.getSignature() + ".", ex);
                 throw ex;
             }
         }
@@ -345,7 +381,7 @@ private void generatePhantom(Object cause) {
         else if (cause instanceof SootMethod)
             _writer.writePhantomMethod(_writer._rep.signature((SootMethod)cause));
         else
-            System.err.println("Ignoring phantom cause: " + cause);
+            logger.error("Ignoring phantom cause: " + cause);
     }
 
     /**
diff --git a/src/main/java/org/clyze/doop/soot/FactWriter.java b/src/main/java/org/clyze/doop/soot/FactWriter.java
diff --git a/src/main/java/org/clyze/doop/soot/Main.java b/src/main/java/org/clyze/doop/soot/Main.java
diff --git a/src/main/resources/log4j2.xml b/src/main/resources/log4j2.xml
