Optimize ByteCodeTranslator IO with caching and buffering

vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeTranslator.java

@@ -23,6 +23,8 @@
 
 package com.codename1.tools.translator;
 
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
 import java.io.DataInputStream;
 import java.io.File;
 import java.io.FileFilter;
@@ -33,6 +35,9 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
+import java.nio.file.Files;
+import java.nio.file.StandardCopyOption;
+import java.nio.file.StandardOpenOption;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashSet;
@@ -236,14 +241,14 @@ private static void handleCleanOutput(ByteCodeTranslator b, File[] sources, File
         b.execute(sources, srcRoot);
 
         File cn1Globals = new File(srcRoot, "cn1_globals.h");
-        copy(ByteCodeTranslator.class.getResourceAsStream("/cn1_globals.h"), new FileOutputStream(cn1Globals));
+        Files.copy(ByteCodeTranslator.class.getResourceAsStream("/cn1_globals.h"), cn1Globals.toPath(), StandardCopyOption.REPLACE_EXISTING);
         if (System.getProperty("INCLUDE_NPE_CHECKS", "false").equals("true")) {
             replaceInFile(cn1Globals, "//#define CN1_INCLUDE_NPE_CHECKS",  "#define CN1_INCLUDE_NPE_CHECKS");
         }
         File xmlvm = new File(srcRoot, "xmlvm.h");
-        copy(ByteCodeTranslator.class.getResourceAsStream("/xmlvm.h"), new FileOutputStream(xmlvm));
+        Files.copy(ByteCodeTranslator.class.getResourceAsStream("/xmlvm.h"), xmlvm.toPath(), StandardCopyOption.REPLACE_EXISTING);
         File nativeMethods = new File(srcRoot, "nativeMethods.m");
-        copy(ByteCodeTranslator.class.getResourceAsStream("/nativeMethods.m"), new FileOutputStream(nativeMethods));
+        Files.copy(ByteCodeTranslator.class.getResourceAsStream("/nativeMethods.m"), nativeMethods.toPath(), StandardCopyOption.REPLACE_EXISTING);
 
         Parser.writeOutput(srcRoot);
 
@@ -268,13 +273,13 @@ private static void handleIosOutput(ByteCodeTranslator b, File[] sources, File d
         launchImageLaunchimage.mkdirs();
         //cleanDir(launchImageLaunchimage);
 
-        copy(ByteCodeTranslator.class.getResourceAsStream("/LaunchImages.json"), new FileOutputStream(new File(launchImageLaunchimage, "Contents.json")));
+        Files.copy(ByteCodeTranslator.class.getResourceAsStream("/LaunchImages.json"), new File(launchImageLaunchimage, "Contents.json").toPath(), StandardCopyOption.REPLACE_EXISTING);
 
         File appIconAppiconset = new File(imagesXcassets, "AppIcon.appiconset");
         appIconAppiconset.mkdirs();
         //cleanDir(appIconAppiconset);
 
-        copy(ByteCodeTranslator.class.getResourceAsStream("/Icons.json"), new FileOutputStream(new File(appIconAppiconset, "Contents.json")));
+        Files.copy(ByteCodeTranslator.class.getResourceAsStream("/Icons.json"), new File(appIconAppiconset, "Contents.json").toPath(), StandardCopyOption.REPLACE_EXISTING);
 
 
         File xcproj = new File(root, appName + ".xcodeproj");
@@ -291,37 +296,37 @@ private static void handleIosOutput(ByteCodeTranslator b, File[] sources, File d
         b.execute(sources, srcRoot);
 
         File cn1Globals = new File(srcRoot, "cn1_globals.h");
-        copy(ByteCodeTranslator.class.getResourceAsStream("/cn1_globals.h"), new FileOutputStream(cn1Globals));
+        Files.copy(ByteCodeTranslator.class.getResourceAsStream("/cn1_globals.h"), cn1Globals.toPath(), StandardCopyOption.REPLACE_EXISTING);
         if (System.getProperty("INCLUDE_NPE_CHECKS", "false").equals("true")) {
             replaceInFile(cn1Globals, "//#define CN1_INCLUDE_NPE_CHECKS",  "#define CN1_INCLUDE_NPE_CHECKS");
         }
         File cn1GlobalsM = new File(srcRoot, "cn1_globals.m");
-        copy(ByteCodeTranslator.class.getResourceAsStream("/cn1_globals.m"), new FileOutputStream(cn1GlobalsM));
+        Files.copy(ByteCodeTranslator.class.getResourceAsStream("/cn1_globals.m"), cn1GlobalsM.toPath(), StandardCopyOption.REPLACE_EXISTING);
         File nativeMethods = new File(srcRoot, "nativeMethods.m");
-        copy(ByteCodeTranslator.class.getResourceAsStream("/nativeMethods.m"), new FileOutputStream(nativeMethods));
+        Files.copy(ByteCodeTranslator.class.getResourceAsStream("/nativeMethods.m"), nativeMethods.toPath(), StandardCopyOption.REPLACE_EXISTING);
 
         if (System.getProperty("USE_RPMALLOC", "false").equals("true")) {
             File malloc = new File(srcRoot, "malloc.c");
-            copy(ByteCodeTranslator.class.getResourceAsStream("/malloc.c"), new FileOutputStream(malloc));
+            Files.copy(ByteCodeTranslator.class.getResourceAsStream("/malloc.c"), malloc.toPath(), StandardCopyOption.REPLACE_EXISTING);
             File rpmalloc = new File(srcRoot, "rpmalloc.c");
-            copy(ByteCodeTranslator.class.getResourceAsStream("/rpmalloc.c"), new FileOutputStream(rpmalloc));
+            Files.copy(ByteCodeTranslator.class.getResourceAsStream("/rpmalloc.c"), rpmalloc.toPath(), StandardCopyOption.REPLACE_EXISTING);
             File rpmalloch = new File(srcRoot, "rpmalloc.h");
-            copy(ByteCodeTranslator.class.getResourceAsStream("/rpmalloc.h"), new FileOutputStream(rpmalloch));
+            Files.copy(ByteCodeTranslator.class.getResourceAsStream("/rpmalloc.h"), rpmalloch.toPath(), StandardCopyOption.REPLACE_EXISTING);
         }
 
         Parser.writeOutput(srcRoot);
 
         File templateInfoPlist = new File(srcRoot, appName + "-Info.plist");
-        copy(ByteCodeTranslator.class.getResourceAsStream("/template/template/template-Info.plist"), new FileOutputStream(templateInfoPlist));
+        Files.copy(ByteCodeTranslator.class.getResourceAsStream("/template/template/template-Info.plist"), templateInfoPlist.toPath(), StandardCopyOption.REPLACE_EXISTING);
 
         File templatePch = new File(srcRoot, appName + "-Prefix.pch");
-        copy(ByteCodeTranslator.class.getResourceAsStream("/template/template/template-Prefix.pch"), new FileOutputStream(templatePch));
+        Files.copy(ByteCodeTranslator.class.getResourceAsStream("/template/template/template-Prefix.pch"), templatePch.toPath(), StandardCopyOption.REPLACE_EXISTING);
 
         File xmlvm = new File(srcRoot, "xmlvm.h");
-        copy(ByteCodeTranslator.class.getResourceAsStream("/xmlvm.h"), new FileOutputStream(xmlvm));
+        Files.copy(ByteCodeTranslator.class.getResourceAsStream("/xmlvm.h"), xmlvm.toPath(), StandardCopyOption.REPLACE_EXISTING);
 
         File projectWorkspaceData = new File(projectXCworkspace, "contents.xcworkspacedata");
-        copy(ByteCodeTranslator.class.getResourceAsStream("/template/template.xcodeproj/project.xcworkspace/contents.xcworkspacedata"), new FileOutputStream(projectWorkspaceData));
+        Files.copy(ByteCodeTranslator.class.getResourceAsStream("/template/template.xcodeproj/project.xcworkspace/contents.xcworkspacedata"), projectWorkspaceData.toPath(), StandardCopyOption.REPLACE_EXISTING);
         replaceInFile(projectWorkspaceData, "KitchenSink", appName);
 
 
@@ -620,11 +625,8 @@ private static String getFileType(String s) {
     //
     private static StringBuilder readFileAsStringBuilder(File sourceFile) throws IOException
     {
-        DataInputStream dis = new DataInputStream(new FileInputStream(sourceFile));
-        byte[] data = new byte[(int)sourceFile.length()];
-        dis.readFully(data);
-        dis.close();
-        StringBuilder b = new StringBuilder(new String(data));
+        byte[] data = Files.readAllBytes(sourceFile.toPath());
+        StringBuilder b = new StringBuilder(new String(data, "UTF-8"));
         return b;
     }
     //
@@ -654,9 +656,7 @@ private static void replaceInFile(File sourceFile, String... values) throws IOEx
         // don't start the output file until all the processing is done
         //
         System.out.println("Rewrite " + sourceFile + " with " + totchanges + " changes");
-        FileWriter fios = new FileWriter(sourceFile);
-        fios.write(str.toString());
-        fios.close();
+        Files.write(sourceFile.toPath(), str.toString().getBytes("UTF-8"), StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE);
     }
 
 
@@ -683,12 +683,25 @@ public static void copy(InputStream i, OutputStream o) throws IOException {
      */
     public static void copy(InputStream i, OutputStream o, int bufferSize) throws IOException {
         try {
+            if (i instanceof FileInputStream && o instanceof FileOutputStream) {
+               // This case should be handled by caller using Files.copy usually, but if streams are passed we can't cast to Path
+               // We can't use Files.copy with streams easily without copying to temp.
+               // So we stick to stream copy but ensure buffering.
+            }
+
+            if(!(i instanceof BufferedInputStream)) {
+                i = new BufferedInputStream(i);
+            }
+            if(!(o instanceof BufferedOutputStream)) {
+                o = new BufferedOutputStream(o);
+            }
             byte[] buffer = new byte[bufferSize];
             int size = i.read(buffer);
             while(size > -1) {
                 o.write(buffer, 0, size);
                 size = i.read(buffer);
             }
+            o.flush();
         } finally {
             cleanup(o);
             cleanup(i);

vm/ByteCodeTranslator/src/com/codename1/tools/translator/Cache.java

@@ -0,0 +1,113 @@
+/*
+ * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Codename One designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Codename One through http://www.codenameone.com/ if you
+ * need additional information or have any questions.
+ */
+package com.codename1.tools.translator;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.Properties;
+
+/**
+ * Manages checksums for generated files to avoid unnecessary writes.
+ */
+public class Cache {
+    private final Properties checksums = new Properties();
+    private final File cacheFile;
+    private final MessageDigest digest;
+
+    public Cache(File outputDir) {
+        this.cacheFile = new File(outputDir, "cn1_checksums.props");
+        try {
+            this.digest = MessageDigest.getInstance("MD5");
+        } catch (NoSuchAlgorithmException ex) {
+            throw new RuntimeException(ex);
+        }
+        load();
+    }
+
+    private void load() {
+        if (cacheFile.exists()) {
+            try {
+                FileInputStream fis = new FileInputStream(cacheFile);
+                BufferedInputStream bis = new BufferedInputStream(fis);
+                checksums.load(bis);
+                bis.close();
+            } catch (IOException err) {
+                // ignore corrupted cache
+                err.printStackTrace();
+            }
+        }
+    }
+
+    public void save() {
+        try {
+            FileOutputStream fos = new FileOutputStream(cacheFile);
+            BufferedOutputStream bos = new BufferedOutputStream(fos);
+            checksums.store(bos, "Codename One Build Checksums");
+            bos.close();
+        } catch (IOException err) {
+            err.printStackTrace();
+        }
+    }
+
+    /**
+     * Checks if the content matches the cache. Updates the cache if it changed.
+     * @param filename The relative filename
+     * @param content The content to write
+     * @return true if the file should be written (changed or new), false otherwise.
+     */
+    public boolean shouldWrite(File destFile, byte[] content) {
+        if (!destFile.exists()) {
+            updateCache(destFile.getName(), content);
+            return true;
+        }
+
+        String hex = calculateHash(content);
+        String existing = checksums.getProperty(destFile.getName());
+
+        if (existing == null || !existing.equals(hex)) {
+            checksums.setProperty(destFile.getName(), hex);
+            return true;
+        }
+        return false;
+    }
+
+    private void updateCache(String key, byte[] content) {
+        checksums.setProperty(key, calculateHash(content));
+    }
+
+    private String calculateHash(byte[] content) {
+        digest.reset();
+        byte[] d = digest.digest(content);
+        StringBuilder sb = new StringBuilder();
+        for (byte b : d) {
+            sb.append(String.format("%02x", b));
+        }
+        return sb.toString();
+    }
+}

vm/ByteCodeTranslator/src/com/codename1/tools/translator/Parser.java

@@ -24,6 +24,8 @@
 package com.codename1.tools.translator;
 
 import java.io.*;
+import java.nio.file.Files;
+import java.nio.file.StandardOpenOption;
 import java.util.*;
 
 import org.objectweb.asm.AnnotationVisitor;
@@ -127,7 +129,7 @@ public static int addToConstantPool(String s) {
 
 
 
-    private static void generateClassAndMethodIndexHeader(File outputDirectory) throws Exception {
+    private static void generateClassAndMethodIndexHeader(File outputDirectory, Cache cache) throws Exception {
         int classOffset = 0;
         int methodOffset = 0;
         ArrayList<BytecodeMethod> methods = new ArrayList<BytecodeMethod>();
@@ -325,12 +327,8 @@ private static void generateClassAndMethodIndexHeader(File outputDirectory) thro
 
         bld.append("\n\n#endif // __CN1_CLASS_METHOD_INDEX_H__\n");        
 
-        FileOutputStream fos = new FileOutputStream(new File(outputDirectory, "cn1_class_method_index.h"));
-        fos.write(bld.toString().getBytes("UTF-8"));
-        fos.close();
-        fos = new FileOutputStream(new File(outputDirectory, "cn1_class_method_index.m"));
-        fos.write(bldM.toString().getBytes("UTF-8"));
-        fos.close();
+        writeIfChanged(new File(outputDirectory, "cn1_class_method_index.h"), bld.toString(), cache);
+        writeIfChanged(new File(outputDirectory, "cn1_class_method_index.m"), bldM.toString(), cache);
     }
 
     private static String encodeString(String con) {
@@ -440,16 +438,18 @@ public static void writeOutput(File outputDirectory) throws Exception {
                 System.out.println("unusued Method cull removed "+neliminated+" methods in "+(dif/1000)+" seconds");
             }
 
-            generateClassAndMethodIndexHeader(outputDirectory);
+            Cache cache = new Cache(outputDirectory);
+            generateClassAndMethodIndexHeader(outputDirectory, cache);
 
             boolean concatenate = "true".equals(System.getProperty("concatenateFiles", "false"));
             ConcatenatingFileOutputStream cos = concatenate ? new ConcatenatingFileOutputStream(outputDirectory) : null;
 
             for(ByteCodeClass bc : classes) {
                 file = bc.getClsName();
-                writeFile(bc, outputDirectory, cos);
+                writeFile(bc, outputDirectory, cos, cache);
             }
             if (cos != null) cos.realClose();
+            cache.save();
 
         } catch(Throwable t) {
             System.out.println("Error while working with the class: " + file);
@@ -626,27 +626,33 @@ private static boolean isMethodUsed(BytecodeMethod m, ByteCodeClass cls) {
         return false;
     }
 
-    private static void writeFile(ByteCodeClass cls, File outputDir, ConcatenatingFileOutputStream writeBufferInstead) throws Exception {
-        OutputStream outMain =
-                writeBufferInstead != null && ByteCodeTranslator.output == ByteCodeTranslator.OutputType.OUTPUT_TYPE_IOS ?
-                        writeBufferInstead :
-                        new FileOutputStream(new File(outputDir, cls.getClsName() + "." + ByteCodeTranslator.output.extension()));
+    private static void writeFile(ByteCodeClass cls, File outputDir, ConcatenatingFileOutputStream writeBufferInstead, Cache cache) throws Exception {
+        if (writeBufferInstead != null && ByteCodeTranslator.output == ByteCodeTranslator.OutputType.OUTPUT_TYPE_IOS) {
+            writeBufferInstead.beginNextFile(cls.getClsName());
+            writeBufferInstead.write(cls.generateCCode(classes).getBytes("UTF-8"));
+            writeBufferInstead.close();
 
-        if (outMain instanceof ConcatenatingFileOutputStream) {
-            ((ConcatenatingFileOutputStream)outMain).beginNextFile(cls.getClsName());
+            // we also need to write the header file for C outputs
+            String headerName = cls.getClsName() + ".h";
+            writeIfChanged(new File(outputDir, headerName), cls.generateCHeader(), cache);
+            return;
         }
+
         if(ByteCodeTranslator.output == ByteCodeTranslator.OutputType.OUTPUT_TYPE_CSHARP) {
-            outMain.write(cls.generateCSharpCode().getBytes());
-            outMain.close();
+            writeIfChanged(new File(outputDir, cls.getClsName() + "." + ByteCodeTranslator.output.extension()), cls.generateCSharpCode(), cache);
         } else {
-            outMain.write(cls.generateCCode(classes).getBytes());
-            outMain.close();
+            writeIfChanged(new File(outputDir, cls.getClsName() + "." + ByteCodeTranslator.output.extension()), cls.generateCCode(classes), cache);
 
             // we also need to write the header file for C outputs
             String headerName = cls.getClsName() + ".h";
-            FileOutputStream outHeader = new FileOutputStream(new File(outputDir, headerName));
-            outHeader.write(cls.generateCHeader().getBytes());
-            outHeader.close();
+            writeIfChanged(new File(outputDir, headerName), cls.generateCHeader(), cache);
+        }
+    }
+
+    private static void writeIfChanged(File dest, String content, Cache cache) throws IOException {
+        byte[] data = content.getBytes("UTF-8");
+        if(cache.shouldWrite(dest, data)) {
+            Files.write(dest.toPath(), data, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE);
         }
     }