improving read performance for compressed files (Z, zip, bz2, gzip, gz) by 300%

Author: michaeldiener

cdm/core/src/main/java/ucar/nc2/NetcdfFile.java

@@ -631,15 +631,15 @@ private static String makeUncompressed(String filename) throws IOException {
 
       try {
         if (suffix.equalsIgnoreCase("Z")) {
-          try (InputStream in = new UncompressInputStream(new FileInputStream(filename))) {
+          try (InputStream in = new UncompressInputStream(new BufferedInputStream(new FileInputStream(filename)))) {
             copy(in, fout, 100000);
           }
           if (debugCompress)
             log.info("uncompressed {} to {}", filename, uncompressedFile);
 
         } else if (suffix.equalsIgnoreCase("zip")) {
 
-          try (ZipInputStream zin = new ZipInputStream(new FileInputStream(filename))) {
+          try (ZipInputStream zin = new ZipInputStream(new BufferedInputStream(new FileInputStream(filename)))) {
             ZipEntry ze = zin.getNextEntry();
             if (ze != null) {
               copy(zin, fout, 100000);
@@ -649,15 +649,15 @@ private static String makeUncompressed(String filename) throws IOException {
           }
 
         } else if (suffix.equalsIgnoreCase("bz2")) {
-          try (InputStream in = new CBZip2InputStream(new FileInputStream(filename), true)) {
+          try (InputStream in = new CBZip2InputStream(new BufferedInputStream(new FileInputStream(filename)), true)) {
             copy(in, fout, 100000);
           }
           if (debugCompress)
             log.info("unbzipped {} to {}", filename, uncompressedFile);
 
         } else if (suffix.equalsIgnoreCase("gzip") || suffix.equalsIgnoreCase("gz")) {
 
-          try (InputStream in = new GZIPInputStream(new FileInputStream(filename))) {
+          try (InputStream in = new GZIPInputStream(new BufferedInputStream(new FileInputStream(filename)))) {
             copy(in, fout, 100000);
           }
 

cdm/core/src/main/java/ucar/nc2/NetcdfFiles.java

@@ -575,7 +575,7 @@ private static String makeUncompressed(String filename) throws Exception {
       try {
         if (suffix.equalsIgnoreCase("Z")) {
           // Z file can only contain one file - copy the whole thing
-          try (InputStream in = new UncompressInputStream(new FileInputStream(baseFilename))) {
+          try (InputStream in = new UncompressInputStream(new BufferedInputStream(new FileInputStream(baseFilename)))) {
             copy(in, fout, 100000);
           }
           if (NetcdfFile.debugCompress) {
@@ -584,7 +584,7 @@ private static String makeUncompressed(String filename) throws Exception {
 
         } else if (suffix.equalsIgnoreCase("zip")) {
           // find specified zip entry, if it exists
-          try (ZipInputStream zin = new ZipInputStream(new FileInputStream(baseFilename))) {
+          try (ZipInputStream zin = new ZipInputStream(new BufferedInputStream(new FileInputStream(baseFilename)))) {
             // If a desired zipentry ID was appended to method's filename parameter, then itempath
             // is of length > 1 and ID starts at itempath char offset 1.
             String itemName = (itempath.length() > 1) ? itempath.substring(1) : "";
@@ -606,15 +606,15 @@ private static String makeUncompressed(String filename) throws Exception {
 
         } else if (suffix.equalsIgnoreCase("bz2")) {
           // bz2 can only contain one file - copy the whole thing
-          try (InputStream in = new CBZip2InputStream(new FileInputStream(baseFilename), true)) {
+          try (InputStream in = new CBZip2InputStream(new BufferedInputStream(new FileInputStream(baseFilename)), true)) {
             copy(in, fout, 100000);
           }
           if (NetcdfFile.debugCompress)
             log.info("unbzipped {} to {}", filename, uncompressedFile);
 
         } else if (suffix.equalsIgnoreCase("gzip") || suffix.equalsIgnoreCase("gz")) {
           // gzip/gz concatenates streams - copy the whole thing
-          try (InputStream in = new GZIPInputStream(new FileInputStream(baseFilename))) {
+          try (InputStream in = new GZIPInputStream(new BufferedInputStream(new FileInputStream(baseFilename)))) {
             copy(in, fout, 100000);
           }