Use a file cache to reduce file open/close operations

Author: blootsvoets

src/main/java/org/radarcns/FileCache.java

@@ -1,8 +1,112 @@
 package org.radarcns;
 
-/**
- * Created by joris on 13/04/2017.
- */
-public class FileCache {
+import java.io.BufferedWriter;
+import java.io.Closeable;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.Flushable;
+import java.io.IOException;
+import java.io.Writer;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
+public class FileCache implements Flushable, Closeable {
+    private static final Logger logger = LoggerFactory.getLogger(FileCache.class);
+
+    private final int maxFiles;
+    private final Map<File, SingleFileCache> caches;
+
+    public FileCache(int maxFiles) {
+        this.maxFiles = maxFiles;
+        this.caches = new HashMap<>();
+    }
+
+    public void appendLine(File file, String data) throws IOException {
+        SingleFileCache cache = caches.get(file);
+        if (cache == null) {
+            if (caches.size() == maxFiles) {
+                ArrayList<SingleFileCache> cacheList = new ArrayList<>(caches.values());
+                Collections.sort(cacheList);
+                for (int i = 0; i < cacheList.size() / 2; i++) {
+                    SingleFileCache rmCache = cacheList.get(i);
+                    caches.remove(rmCache.getFile());
+                    rmCache.close();
+                }
+            }
+
+            File dir = file.getParentFile();
+            if (!file.getParentFile().exists()){
+                if (dir.mkdirs()) {
+                    logger.info("Created directory: {}", dir.getAbsolutePath());
+                } else {
+                    logger.warn("FAILED to create directory: " + dir.getAbsolutePath());
+                }
+            }
+
+            cache = new SingleFileCache(file);
+            caches.put(file, cache);
+        }
+        cache.appendLine(data);
+    }
+
+    @Override
+    public void flush() throws IOException {
+        for (SingleFileCache cache : caches.values()) {
+            cache.flush();
+        }
+    }
+
+    @Override
+    public void close() throws IOException {
+        try {
+            for (SingleFileCache cache : caches.values()) {
+                cache.close();
+            }
+        } finally {
+            caches.clear();
+        }
+    }
+
+    private static class SingleFileCache implements Closeable, Flushable, Comparable<SingleFileCache> {
+        private final BufferedWriter bufferedWriter;
+        private final Writer fileWriter;
+        private final File file;
+        private long lastUse;
+
+        private SingleFileCache(File file) throws IOException {
+            this.file = file;
+            this.fileWriter = new FileWriter(file, true);
+            this.bufferedWriter = new BufferedWriter(fileWriter);
+        }
+
+        private void appendLine(String data) throws IOException {
+            bufferedWriter.write(data);
+            bufferedWriter.write('\n');
+            lastUse = System.nanoTime();
+        }
+
+        @Override
+        public void close() throws IOException {
+            bufferedWriter.close();
+            fileWriter.close();
+        }
+
+        @Override
+        public void flush() throws IOException {
+            bufferedWriter.flush();
+        }
+
+        @Override
+        public int compareTo(SingleFileCache other) {
+            return Long.compare(lastUse, other.lastUse);
+        }
+
+        private File getFile() {
+            return file;
+        }
+    }
 }

src/main/java/org/radarcns/Frequency.java

@@ -3,27 +3,27 @@
 import org.apache.commons.collections.MapIterator;
 import org.apache.commons.collections.keyvalue.MultiKey;
 import org.apache.commons.collections.map.MultiKeyMap;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
 
 import java.io.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 
 public class Frequency {
-    private static final Logger logger = LogManager.getLogger(RestructureAvroRecords.class);
+    private static final Logger logger = LoggerFactory.getLogger(Frequency.class);
 
     private MultiKeyMap bins = new MultiKeyMap();
-    private String binFilePath;
+    private File binFilePath;
 
-    public void setBinFilePath(String binFilePath) {
+    public void setBinFilePath(File binFilePath) {
         this.binFilePath = binFilePath;
     }
 
     public MultiKeyMap getBins() {
         return bins;
     }
 
-    public void addToBin(String topicName, String id, String timestamp, Integer countToAdd) {
+    public void addToBin(String topicName, String id, String timestamp, int countToAdd) {
         Integer count = (Integer) bins.get(topicName, id, timestamp);
         if (count == null) {
             bins.put(topicName, id, timestamp, countToAdd);
@@ -32,7 +32,7 @@ public void addToBin(String topicName, String id, String timestamp, Integer coun
         }
     }
 
-    public void addToBin(String topicName, String id, Double time, Integer countToAdd) {
+    public void addToBin(String topicName, String id, Double time, int countToAdd) {
         // Hour resolution
         String hourlyTimestamp = RestructureAvroRecords.createHourTimestamp(time);