Skip to content

Commit 3c3a5fe

Browse files
google-labs-jules[bot]google-labs-jules[bot]
authored
Optimize ByteCodeTranslator IO with caching and buffering (#4295)
* Optimize ByteCodeTranslator IO with caching and buffering - Implemented `writeIfChanged` in `Parser.java` to skip writing files if content is identical (Smart Writes). - Added `BufferedInputStream` and `BufferedOutputStream` wrappers in `ByteCodeTranslator.java` and `Parser.java` to improve IO performance. - Added explicit flush in `ByteCodeTranslator.copy` to prevent data loss. - Used UTF-8 encoding explicitly for generated files. * Optimize ByteCodeTranslator IO with caching and NIO - Implemented `Cache` class to skip writing generated files if their MD5 hash matches (Smart Writes). - Updated `Parser.java` to use `Cache` and NIO `Files.write` for file generation. - Updated `ByteCodeTranslator.java` to use NIO `Files.copy` and `Files.write` for efficient file copying and modification. - Standardized on UTF-8 encoding for all generated files. - Improved IO performance by reducing unnecessary writes and using buffered/NIO operations. --------- Co-authored-by: google-labs-jules[bot] <161369871+google-labs-jules[bot]@users.noreply.github.com>
1 parent a98a485 commit 3c3a5fe

File tree

3 files changed

+178
-46
lines changed

3 files changed

+178
-46
lines changed

‎vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeTranslator.java‎

Lines changed: 36 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,8 @@
2323

2424
package com.codename1.tools.translator;
2525

26+
import java.io.BufferedInputStream;
27+
import java.io.BufferedOutputStream;
2628
import java.io.DataInputStream;
2729
import java.io.File;
2830
import java.io.FileFilter;
@@ -33,6 +35,9 @@
3335
import java.io.IOException;
3436
import java.io.InputStream;
3537
import java.io.OutputStream;
38+
import java.nio.file.Files;
39+
import java.nio.file.StandardCopyOption;
40+
import java.nio.file.StandardOpenOption;
3641
import java.util.ArrayList;
3742
import java.util.Arrays;
3843
import java.util.HashSet;
@@ -236,14 +241,14 @@ private static void handleCleanOutput(ByteCodeTranslator b, File[] sources, File
236241
b.execute(sources, srcRoot);
237242

238243
File cn1Globals = new File(srcRoot, "cn1_globals.h");
239-
copy(ByteCodeTranslator.class.getResourceAsStream("/cn1_globals.h"), new FileOutputStream(cn1Globals));
244+
Files.copy(ByteCodeTranslator.class.getResourceAsStream("/cn1_globals.h"), cn1Globals.toPath(), StandardCopyOption.REPLACE_EXISTING);
240245
if (System.getProperty("INCLUDE_NPE_CHECKS", "false").equals("true")) {
241246
replaceInFile(cn1Globals, "//#define CN1_INCLUDE_NPE_CHECKS", "#define CN1_INCLUDE_NPE_CHECKS");
242247
}
243248
File xmlvm = new File(srcRoot, "xmlvm.h");
244-
copy(ByteCodeTranslator.class.getResourceAsStream("/xmlvm.h"), new FileOutputStream(xmlvm));
249+
Files.copy(ByteCodeTranslator.class.getResourceAsStream("/xmlvm.h"), xmlvm.toPath(), StandardCopyOption.REPLACE_EXISTING);
245250
File nativeMethods = new File(srcRoot, "nativeMethods.m");
246-
copy(ByteCodeTranslator.class.getResourceAsStream("/nativeMethods.m"), new FileOutputStream(nativeMethods));
251+
Files.copy(ByteCodeTranslator.class.getResourceAsStream("/nativeMethods.m"), nativeMethods.toPath(), StandardCopyOption.REPLACE_EXISTING);
247252

248253
Parser.writeOutput(srcRoot);
249254

@@ -268,13 +273,13 @@ private static void handleIosOutput(ByteCodeTranslator b, File[] sources, File d
268273
launchImageLaunchimage.mkdirs();
269274
//cleanDir(launchImageLaunchimage);
270275

271-
copy(ByteCodeTranslator.class.getResourceAsStream("/LaunchImages.json"), new FileOutputStream(new File(launchImageLaunchimage, "Contents.json")));
276+
Files.copy(ByteCodeTranslator.class.getResourceAsStream("/LaunchImages.json"), new File(launchImageLaunchimage, "Contents.json").toPath(), StandardCopyOption.REPLACE_EXISTING);
272277

273278
File appIconAppiconset = new File(imagesXcassets, "AppIcon.appiconset");
274279
appIconAppiconset.mkdirs();
275280
//cleanDir(appIconAppiconset);
276281

277-
copy(ByteCodeTranslator.class.getResourceAsStream("/Icons.json"), new FileOutputStream(new File(appIconAppiconset, "Contents.json")));
282+
Files.copy(ByteCodeTranslator.class.getResourceAsStream("/Icons.json"), new File(appIconAppiconset, "Contents.json").toPath(), StandardCopyOption.REPLACE_EXISTING);
278283

279284

280285
File xcproj = new File(root, appName + ".xcodeproj");
@@ -291,37 +296,37 @@ private static void handleIosOutput(ByteCodeTranslator b, File[] sources, File d
291296
b.execute(sources, srcRoot);
292297

293298
File cn1Globals = new File(srcRoot, "cn1_globals.h");
294-
copy(ByteCodeTranslator.class.getResourceAsStream("/cn1_globals.h"), new FileOutputStream(cn1Globals));
299+
Files.copy(ByteCodeTranslator.class.getResourceAsStream("/cn1_globals.h"), cn1Globals.toPath(), StandardCopyOption.REPLACE_EXISTING);
295300
if (System.getProperty("INCLUDE_NPE_CHECKS", "false").equals("true")) {
296301
replaceInFile(cn1Globals, "//#define CN1_INCLUDE_NPE_CHECKS", "#define CN1_INCLUDE_NPE_CHECKS");
297302
}
298303
File cn1GlobalsM = new File(srcRoot, "cn1_globals.m");
299-
copy(ByteCodeTranslator.class.getResourceAsStream("/cn1_globals.m"), new FileOutputStream(cn1GlobalsM));
304+
Files.copy(ByteCodeTranslator.class.getResourceAsStream("/cn1_globals.m"), cn1GlobalsM.toPath(), StandardCopyOption.REPLACE_EXISTING);
300305
File nativeMethods = new File(srcRoot, "nativeMethods.m");
301-
copy(ByteCodeTranslator.class.getResourceAsStream("/nativeMethods.m"), new FileOutputStream(nativeMethods));
306+
Files.copy(ByteCodeTranslator.class.getResourceAsStream("/nativeMethods.m"), nativeMethods.toPath(), StandardCopyOption.REPLACE_EXISTING);
302307

303308
if (System.getProperty("USE_RPMALLOC", "false").equals("true")) {
304309
File malloc = new File(srcRoot, "malloc.c");
305-
copy(ByteCodeTranslator.class.getResourceAsStream("/malloc.c"), new FileOutputStream(malloc));
310+
Files.copy(ByteCodeTranslator.class.getResourceAsStream("/malloc.c"), malloc.toPath(), StandardCopyOption.REPLACE_EXISTING);
306311
File rpmalloc = new File(srcRoot, "rpmalloc.c");
307-
copy(ByteCodeTranslator.class.getResourceAsStream("/rpmalloc.c"), new FileOutputStream(rpmalloc));
312+
Files.copy(ByteCodeTranslator.class.getResourceAsStream("/rpmalloc.c"), rpmalloc.toPath(), StandardCopyOption.REPLACE_EXISTING);
308313
File rpmalloch = new File(srcRoot, "rpmalloc.h");
309-
copy(ByteCodeTranslator.class.getResourceAsStream("/rpmalloc.h"), new FileOutputStream(rpmalloch));
314+
Files.copy(ByteCodeTranslator.class.getResourceAsStream("/rpmalloc.h"), rpmalloch.toPath(), StandardCopyOption.REPLACE_EXISTING);
310315
}
311316

312317
Parser.writeOutput(srcRoot);
313318

314319
File templateInfoPlist = new File(srcRoot, appName + "-Info.plist");
315-
copy(ByteCodeTranslator.class.getResourceAsStream("/template/template/template-Info.plist"), new FileOutputStream(templateInfoPlist));
320+
Files.copy(ByteCodeTranslator.class.getResourceAsStream("/template/template/template-Info.plist"), templateInfoPlist.toPath(), StandardCopyOption.REPLACE_EXISTING);
316321

317322
File templatePch = new File(srcRoot, appName + "-Prefix.pch");
318-
copy(ByteCodeTranslator.class.getResourceAsStream("/template/template/template-Prefix.pch"), new FileOutputStream(templatePch));
323+
Files.copy(ByteCodeTranslator.class.getResourceAsStream("/template/template/template-Prefix.pch"), templatePch.toPath(), StandardCopyOption.REPLACE_EXISTING);
319324

320325
File xmlvm = new File(srcRoot, "xmlvm.h");
321-
copy(ByteCodeTranslator.class.getResourceAsStream("/xmlvm.h"), new FileOutputStream(xmlvm));
326+
Files.copy(ByteCodeTranslator.class.getResourceAsStream("/xmlvm.h"), xmlvm.toPath(), StandardCopyOption.REPLACE_EXISTING);
322327

323328
File projectWorkspaceData = new File(projectXCworkspace, "contents.xcworkspacedata");
324-
copy(ByteCodeTranslator.class.getResourceAsStream("/template/template.xcodeproj/project.xcworkspace/contents.xcworkspacedata"), new FileOutputStream(projectWorkspaceData));
329+
Files.copy(ByteCodeTranslator.class.getResourceAsStream("/template/template.xcodeproj/project.xcworkspace/contents.xcworkspacedata"), projectWorkspaceData.toPath(), StandardCopyOption.REPLACE_EXISTING);
325330
replaceInFile(projectWorkspaceData, "KitchenSink", appName);
326331

327332

@@ -620,11 +625,8 @@ private static String getFileType(String s) {
620625
//
621626
private static StringBuilder readFileAsStringBuilder(File sourceFile) throws IOException
622627
{
623-
DataInputStream dis = new DataInputStream(new FileInputStream(sourceFile));
624-
byte[] data = new byte[(int)sourceFile.length()];
625-
dis.readFully(data);
626-
dis.close();
627-
StringBuilder b = new StringBuilder(new String(data));
628+
byte[] data = Files.readAllBytes(sourceFile.toPath());
629+
StringBuilder b = new StringBuilder(new String(data, "UTF-8"));
628630
return b;
629631
}
630632
//
@@ -654,9 +656,7 @@ private static void replaceInFile(File sourceFile, String... values) throws IOEx
654656
// don't start the output file until all the processing is done
655657
//
656658
System.out.println("Rewrite " + sourceFile + " with " + totchanges + " changes");
657-
FileWriter fios = new FileWriter(sourceFile);
658-
fios.write(str.toString());
659-
fios.close();
659+
Files.write(sourceFile.toPath(), str.toString().getBytes("UTF-8"), StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE);
660660
}
661661

662662

@@ -683,12 +683,25 @@ public static void copy(InputStream i, OutputStream o) throws IOException {
683683
*/
684684
public static void copy(InputStream i, OutputStream o, int bufferSize) throws IOException {
685685
try {
686+
if (i instanceof FileInputStream && o instanceof FileOutputStream) {
687+
// This case should be handled by caller using Files.copy usually, but if streams are passed we can't cast to Path
688+
// We can't use Files.copy with streams easily without copying to temp.
689+
// So we stick to stream copy but ensure buffering.
690+
}
691+
692+
if(!(i instanceof BufferedInputStream)) {
693+
i = new BufferedInputStream(i);
694+
}
695+
if(!(o instanceof BufferedOutputStream)) {
696+
o = new BufferedOutputStream(o);
697+
}
686698
byte[] buffer = new byte[bufferSize];
687699
int size = i.read(buffer);
688700
while(size > -1) {
689701
o.write(buffer, 0, size);
690702
size = i.read(buffer);
691703
}
704+
o.flush();
692705
} finally {
693706
cleanup(o);
694707
cleanup(i);

‎vm/ByteCodeTranslator/src/com/codename1/tools/translator/Cache.java‎

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
/*
2+
* Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved.
3+
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4+
* This code is free software; you can redistribute it and/or modify it
5+
* under the terms of the GNU General Public License version 2 only, as
6+
* published by the Free Software Foundation. Codename One designates this
7+
* particular file as subject to the "Classpath" exception as provided
8+
* by Oracle in the LICENSE file that accompanied this code.
9+
*
10+
* This code is distributed in the hope that it will be useful, but WITHOUT
11+
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12+
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13+
* version 2 for more details (a copy is included in the LICENSE file that
14+
* accompanied this code).
15+
*
16+
* You should have received a copy of the GNU General Public License version
17+
* 2 along with this work; if not, write to the Free Software Foundation,
18+
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19+
*
20+
* Please contact Codename One through http://www.codenameone.com/ if you
21+
* need additional information or have any questions.
22+
*/
23+
package com.codename1.tools.translator;
24+
25+
import java.io.BufferedInputStream;
26+
import java.io.BufferedOutputStream;
27+
import java.io.File;
28+
import java.io.FileInputStream;
29+
import java.io.FileOutputStream;
30+
import java.io.IOException;
31+
import java.security.MessageDigest;
32+
import java.security.NoSuchAlgorithmException;
33+
import java.util.Properties;
34+
35+
/**
36+
* Manages checksums for generated files to avoid unnecessary writes.
37+
*/
38+
public class Cache {
39+
private final Properties checksums = new Properties();
40+
private final File cacheFile;
41+
private final MessageDigest digest;
42+
43+
public Cache(File outputDir) {
44+
this.cacheFile = new File(outputDir, "cn1_checksums.props");
45+
try {
46+
this.digest = MessageDigest.getInstance("MD5");
47+
} catch (NoSuchAlgorithmException ex) {
48+
throw new RuntimeException(ex);
49+
}
50+
load();
51+
}
52+
53+
private void load() {
54+
if (cacheFile.exists()) {
55+
try {
56+
FileInputStream fis = new FileInputStream(cacheFile);
57+
BufferedInputStream bis = new BufferedInputStream(fis);
58+
checksums.load(bis);
59+
bis.close();
60+
} catch (IOException err) {
61+
// ignore corrupted cache
62+
err.printStackTrace();
63+
}
64+
}
65+
}
66+
67+
public void save() {
68+
try {
69+
FileOutputStream fos = new FileOutputStream(cacheFile);
70+
BufferedOutputStream bos = new BufferedOutputStream(fos);
71+
checksums.store(bos, "Codename One Build Checksums");
72+
bos.close();
73+
} catch (IOException err) {
74+
err.printStackTrace();
75+
}
76+
}
77+
78+
/**
79+
* Checks if the content matches the cache. Updates the cache if it changed.
80+
* @param filename The relative filename
81+
* @param content The content to write
82+
* @return true if the file should be written (changed or new), false otherwise.
83+
*/
84+
public boolean shouldWrite(File destFile, byte[] content) {
85+
if (!destFile.exists()) {
86+
updateCache(destFile.getName(), content);
87+
return true;
88+
}
89+
90+
String hex = calculateHash(content);
91+
String existing = checksums.getProperty(destFile.getName());
92+
93+
if (existing == null || !existing.equals(hex)) {
94+
checksums.setProperty(destFile.getName(), hex);
95+
return true;
96+
}
97+
return false;
98+
}
99+
100+
private void updateCache(String key, byte[] content) {
101+
checksums.setProperty(key, calculateHash(content));
102+
}
103+
104+
private String calculateHash(byte[] content) {
105+
digest.reset();
106+
byte[] d = digest.digest(content);
107+
StringBuilder sb = new StringBuilder();
108+
for (byte b : d) {
109+
sb.append(String.format("%02x", b));
110+
}
111+
return sb.toString();
112+
}
113+
}

‎vm/ByteCodeTranslator/src/com/codename1/tools/translator/Parser.java‎

Lines changed: 29 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,8 @@
2424
package com.codename1.tools.translator;
2525

2626
import java.io.*;
27+
import java.nio.file.Files;
28+
import java.nio.file.StandardOpenOption;
2729
import java.util.*;
2830

2931
import org.objectweb.asm.AnnotationVisitor;
@@ -127,7 +129,7 @@ public static int addToConstantPool(String s) {
127129

128130

129131

130-
private static void generateClassAndMethodIndexHeader(File outputDirectory) throws Exception {
132+
private static void generateClassAndMethodIndexHeader(File outputDirectory, Cache cache) throws Exception {
131133
int classOffset = 0;
132134
int methodOffset = 0;
133135
ArrayList<BytecodeMethod> methods = new ArrayList<BytecodeMethod>();
@@ -325,12 +327,8 @@ private static void generateClassAndMethodIndexHeader(File outputDirectory) thro
325327

326328
bld.append("\n\n#endif // __CN1_CLASS_METHOD_INDEX_H__\n");
327329

328-
FileOutputStream fos = new FileOutputStream(new File(outputDirectory, "cn1_class_method_index.h"));
329-
fos.write(bld.toString().getBytes("UTF-8"));
330-
fos.close();
331-
fos = new FileOutputStream(new File(outputDirectory, "cn1_class_method_index.m"));
332-
fos.write(bldM.toString().getBytes("UTF-8"));
333-
fos.close();
330+
writeIfChanged(new File(outputDirectory, "cn1_class_method_index.h"), bld.toString(), cache);
331+
writeIfChanged(new File(outputDirectory, "cn1_class_method_index.m"), bldM.toString(), cache);
334332
}
335333

336334
private static String encodeString(String con) {
@@ -440,16 +438,18 @@ public static void writeOutput(File outputDirectory) throws Exception {
440438
System.out.println("unusued Method cull removed "+neliminated+" methods in "+(dif/1000)+" seconds");
441439
}
442440

443-
generateClassAndMethodIndexHeader(outputDirectory);
441+
Cache cache = new Cache(outputDirectory);
442+
generateClassAndMethodIndexHeader(outputDirectory, cache);
444443

445444
boolean concatenate = "true".equals(System.getProperty("concatenateFiles", "false"));
446445
ConcatenatingFileOutputStream cos = concatenate ? new ConcatenatingFileOutputStream(outputDirectory) : null;
447446

448447
for(ByteCodeClass bc : classes) {
449448
file = bc.getClsName();
450-
writeFile(bc, outputDirectory, cos);
449+
writeFile(bc, outputDirectory, cos, cache);
451450
}
452451
if (cos != null) cos.realClose();
452+
cache.save();
453453

454454
} catch(Throwable t) {
455455
System.out.println("Error while working with the class: " + file);
@@ -626,27 +626,33 @@ private static boolean isMethodUsed(BytecodeMethod m, ByteCodeClass cls) {
626626
return false;
627627
}
628628

629-
private static void writeFile(ByteCodeClass cls, File outputDir, ConcatenatingFileOutputStream writeBufferInstead) throws Exception {
630-
OutputStream outMain =
631-
writeBufferInstead != null && ByteCodeTranslator.output == ByteCodeTranslator.OutputType.OUTPUT_TYPE_IOS ?
632-
writeBufferInstead :
633-
new FileOutputStream(new File(outputDir, cls.getClsName() + "." + ByteCodeTranslator.output.extension()));
629+
private static void writeFile(ByteCodeClass cls, File outputDir, ConcatenatingFileOutputStream writeBufferInstead, Cache cache) throws Exception {
630+
if (writeBufferInstead != null && ByteCodeTranslator.output == ByteCodeTranslator.OutputType.OUTPUT_TYPE_IOS) {
631+
writeBufferInstead.beginNextFile(cls.getClsName());
632+
writeBufferInstead.write(cls.generateCCode(classes).getBytes("UTF-8"));
633+
writeBufferInstead.close();
634634

635-
if (outMain instanceof ConcatenatingFileOutputStream) {
636-
((ConcatenatingFileOutputStream)outMain).beginNextFile(cls.getClsName());
635+
// we also need to write the header file for C outputs
636+
String headerName = cls.getClsName() + ".h";
637+
writeIfChanged(new File(outputDir, headerName), cls.generateCHeader(), cache);
638+
return;
637639
}
640+
638641
if(ByteCodeTranslator.output == ByteCodeTranslator.OutputType.OUTPUT_TYPE_CSHARP) {
639-
outMain.write(cls.generateCSharpCode().getBytes());
640-
outMain.close();
642+
writeIfChanged(new File(outputDir, cls.getClsName() + "." + ByteCodeTranslator.output.extension()), cls.generateCSharpCode(), cache);
641643
} else {
642-
outMain.write(cls.generateCCode(classes).getBytes());
643-
outMain.close();
644+
writeIfChanged(new File(outputDir, cls.getClsName() + "." + ByteCodeTranslator.output.extension()), cls.generateCCode(classes), cache);
644645

645646
// we also need to write the header file for C outputs
646647
String headerName = cls.getClsName() + ".h";
647-
FileOutputStream outHeader = new FileOutputStream(new File(outputDir, headerName));
648-
outHeader.write(cls.generateCHeader().getBytes());
649-
outHeader.close();
648+
writeIfChanged(new File(outputDir, headerName), cls.generateCHeader(), cache);
649+
}
650+
}
651+
652+
private static void writeIfChanged(File dest, String content, Cache cache) throws IOException {
653+
byte[] data = content.getBytes("UTF-8");
654+
if(cache.shouldWrite(dest, data)) {
655+
Files.write(dest.toPath(), data, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE);
650656
}
651657
}
652658

0 commit comments

Comments
 (0)