Tested FileCache

Author: blootsvoets

src/main/java/org/radarcns/RestructureAvroRecords.java

@@ -19,6 +19,7 @@
 import org.apache.hadoop.fs.LocatedFileStatus;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.RemoteIterator;
+import org.radarcns.util.FileCache;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 

src/main/java/org/radarcns/FileCache.java renamed to src/main/java/org/radarcns/util/FileCache.java

@@ -1,4 +1,4 @@
-package org.radarcns;
+package org.radarcns.util;
 
 import java.io.BufferedWriter;
 import java.io.Closeable;
@@ -14,6 +14,10 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+/**
+ * Caches open file handles. If more than the limit is cached, the half of the files that were used
+ * the longest ago cache are evicted from cache.
+ */
 public class FileCache implements Flushable, Closeable {
     private static final Logger logger = LoggerFactory.getLogger(FileCache.class);
 
@@ -22,35 +26,56 @@ public class FileCache implements Flushable, Closeable {
 
     public FileCache(int maxFiles) {
         this.maxFiles = maxFiles;
-        this.caches = new HashMap<>();
+        this.caches = new HashMap<>(maxFiles * 4 / 3 + 1);
     }
 
-    public void appendLine(File file, String data) throws IOException {
+    /**
+     * Append a line to given file. This will append a line ending to given data.
+     * If the file handle and writer are already open in this cache,
+     * those will be used. Otherwise, the file will be opened and the file handle cached.
+     *
+     * @param file file to append data to
+     * @param data data without line ending
+     * @return true if the cache was used, false if a new file was opened.
+     * @throws IOException when failing to open a file or writing to it.
+     */
+    public boolean appendLine(File file, String data) throws IOException {
         SingleFileCache cache = caches.get(file);
-        if (cache == null) {
-            if (caches.size() == maxFiles) {
-                ArrayList<SingleFileCache> cacheList = new ArrayList<>(caches.values());
-                Collections.sort(cacheList);
-                for (int i = 0; i < cacheList.size() / 2; i++) {
-                    SingleFileCache rmCache = cacheList.get(i);
-                    caches.remove(rmCache.getFile());
-                    rmCache.close();
-                }
-            }
+        if (cache != null) {
+            cache.appendLine(data);
+            return true;
+        } else {
+            ensureCapacity();
 
             File dir = file.getParentFile();
             if (!file.getParentFile().exists()){
                 if (dir.mkdirs()) {
-                    logger.info("Created directory: {}", dir.getAbsolutePath());
+                    logger.debug("Created directory: {}", dir.getAbsolutePath());
                 } else {
-                    logger.warn("FAILED to create directory: " + dir.getAbsolutePath());
+                    logger.warn("FAILED to create directory: {}", dir.getAbsolutePath());
                 }
             }
 
             cache = new SingleFileCache(file);
             caches.put(file, cache);
+            cache.appendLine(data);
+            return false;
+        }
+    }
+
+    /**
+     * Ensure that a new filecache can be added. Evict files used longest ago from cache if needed.
+     */
+    private void ensureCapacity() throws IOException {
+        if (caches.size() == maxFiles) {
+            ArrayList<SingleFileCache> cacheList = new ArrayList<>(caches.values());
+            Collections.sort(cacheList);
+            for (int i = 0; i < cacheList.size() / 2; i++) {
+                SingleFileCache rmCache = cacheList.get(i);
+                caches.remove(rmCache.getFile());
+                rmCache.close();
+            }
         }
-        cache.appendLine(data);
     }
 
     @Override

src/test/java/org/radarcns/util/FileCacheTest.java

@@ -0,0 +1,46 @@
+package org.radarcns.util;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+public class FileCacheTest {
+    @Rule
+    public TemporaryFolder folder = new TemporaryFolder();
+
+    @Test
+    public void appendLine() throws IOException {
+        File f1 = folder.newFile();
+        File f2 = folder.newFile();
+        File f3 = folder.newFile();
+        File d4 = folder.newFolder();
+        File f4 = new File(d4, "f4.txt");
+
+        assertTrue(f1.delete());
+        assertTrue(d4.delete());
+
+        try (FileCache cache = new FileCache(2)) {
+            assertFalse(cache.appendLine(f1, "something"));
+            assertTrue(cache.appendLine(f1, "somethingElse"));
+            assertFalse(cache.appendLine(f2, "something"));
+            assertTrue(cache.appendLine(f1, "third"));
+            assertFalse(cache.appendLine(f3, "f3"));
+            assertFalse(cache.appendLine(f2, "f2"));
+            assertTrue(cache.appendLine(f3, "f3"));
+            assertFalse(cache.appendLine(f4, "f4"));
+            assertTrue(cache.appendLine(f3, "f3"));
+        }
+
+        assertEquals("something\nsomethingElse\nthird\n", new String(Files.readAllBytes(f1.toPath())));
+        assertEquals("something\nf2\n", new String(Files.readAllBytes(f2.toPath())));
+        assertEquals("f3\nf3\nf3\n", new String(Files.readAllBytes(f3.toPath())));
+        assertEquals("f4\n", new String(Files.readAllBytes(f4.toPath())));
+    }
+}