|
16 | 16 |
|
17 | 17 | package org.radarcns.util; |
18 | 18 |
|
19 | | -import java.io.BufferedWriter; |
| 19 | +import java.io.BufferedOutputStream; |
20 | 20 | import java.io.Closeable; |
21 | 21 | import java.io.File; |
22 | | -import java.io.FileWriter; |
| 22 | +import java.io.FileOutputStream; |
23 | 23 | import java.io.Flushable; |
24 | 24 | import java.io.IOException; |
| 25 | +import java.io.OutputStream; |
| 26 | +import java.io.OutputStreamWriter; |
25 | 27 | import java.io.Writer; |
26 | | -import java.util.ArrayList; |
27 | | -import java.util.Collections; |
28 | | -import java.util.HashMap; |
29 | | -import java.util.Map; |
| 28 | +import java.util.zip.GZIPOutputStream; |
30 | 29 | import javax.annotation.Nonnull; |
31 | 30 | import org.apache.avro.generic.GenericRecord; |
32 | | -import org.slf4j.Logger; |
33 | | -import org.slf4j.LoggerFactory; |
34 | 31 |
|
35 | | -/** |
36 | | - * Caches open file handles. If more than the limit is cached, the half of the files that were used |
37 | | - * the longest ago cache are evicted from cache. |
38 | | - */ |
39 | | -public class FileCache implements Flushable, Closeable { |
40 | | - private static final Logger logger = LoggerFactory.getLogger(FileCache.class); |
41 | | - |
42 | | - private RecordConverterFactory converterFactory; |
43 | | - private final int maxFiles; |
44 | | - private final Map<File, SingleFileCache> caches; |
45 | | - |
46 | | - public FileCache(RecordConverterFactory converterFactory, int maxFiles) { |
47 | | - this.converterFactory = converterFactory; |
48 | | - this.maxFiles = maxFiles; |
49 | | - this.caches = new HashMap<>(maxFiles * 4 / 3 + 1); |
50 | | - } |
| 32 | +/** Keeps file handles of a file. */ |
| 33 | +public class FileCache implements Closeable, Flushable, Comparable<FileCache> { |
| 34 | + private final OutputStream[] streams; |
| 35 | + private final Writer writer; |
| 36 | + private final RecordConverter recordConverter; |
| 37 | + private final File file; |
| 38 | + private long lastUse; |
51 | 39 |
|
52 | 40 | /** |
53 | | - * Append a record to given file. If the file handle and writer are already open in this cache, |
54 | | - * those will be used. Otherwise, the file will be opened and the file handle cached. |
55 | | - * |
56 | | - * @param file file to append data to |
57 | | - * @param record data |
58 | | - * @return true if the cache was used, false if a new file was opened. |
59 | | - * @throws IOException when failing to open a file or writing to it. |
| 41 | + * File cache of given file, using given converter factory. |
| 42 | + * @param converterFactory converter factory to create a converter to write files with. |
| 43 | + * @param file file to cache. |
| 44 | + * @param record example record to create converter from, this is not written to file. |
| 45 | + * @param gzip whether to gzip the records |
| 46 | + * @throws IOException |
60 | 47 | */ |
61 | | - public boolean writeRecord(File file, GenericRecord record) throws IOException { |
62 | | - SingleFileCache cache = caches.get(file); |
63 | | - if (cache != null) { |
64 | | - cache.writeRecord(record); |
65 | | - return true; |
66 | | - } else { |
67 | | - ensureCapacity(); |
| 48 | + public FileCache(RecordConverterFactory converterFactory, File file, |
| 49 | + GenericRecord record, boolean gzip) throws IOException { |
| 50 | + this.file = file; |
| 51 | + boolean fileIsNew = !file.exists() || file.length() == 0; |
| 52 | + |
| 53 | + this.streams = new OutputStream[gzip ? 3 : 2]; |
| 54 | + this.streams[0] = new FileOutputStream(file, true); |
| 55 | + this.streams[1] = new BufferedOutputStream(this.streams[0]); |
| 56 | + if (gzip) { |
| 57 | + this.streams[2] = new GZIPOutputStream(this.streams[1]); |
| 58 | + } |
68 | 59 |
|
69 | | - File dir = file.getParentFile(); |
70 | | - if (!dir.exists()){ |
71 | | - if (dir.mkdirs()) { |
72 | | - logger.debug("Created directory: {}", dir.getAbsolutePath()); |
73 | | - } else { |
74 | | - logger.warn("FAILED to create directory: {}", dir.getAbsolutePath()); |
75 | | - } |
76 | | - } |
| 60 | + this.writer = new OutputStreamWriter(this.streams[this.streams.length - 1]); |
| 61 | + this.recordConverter = converterFactory.converterFor(writer, record, fileIsNew); |
| 62 | + } |
77 | 63 |
|
78 | | - cache = new SingleFileCache(file, record); |
79 | | - caches.put(file, cache); |
80 | | - cache.writeRecord(record); |
81 | | - return false; |
82 | | - } |
| 64 | + /** Write a record to the cache. */ |
| 65 | + public void writeRecord(GenericRecord record) throws IOException { |
| 66 | + this.recordConverter.writeRecord(record); |
| 67 | + lastUse = System.nanoTime(); |
83 | 68 | } |
84 | 69 |
|
85 | | - /** |
86 | | - * Ensure that a new filecache can be added. Evict files used longest ago from cache if needed. |
87 | | - */ |
88 | | - private void ensureCapacity() throws IOException { |
89 | | - if (caches.size() == maxFiles) { |
90 | | - ArrayList<SingleFileCache> cacheList = new ArrayList<>(caches.values()); |
91 | | - Collections.sort(cacheList); |
92 | | - for (int i = 0; i < cacheList.size() / 2; i++) { |
93 | | - SingleFileCache rmCache = cacheList.get(i); |
94 | | - caches.remove(rmCache.getFile()); |
95 | | - rmCache.close(); |
96 | | - } |
| 70 | + @Override |
| 71 | + public void close() throws IOException { |
| 72 | + recordConverter.close(); |
| 73 | + writer.close(); |
| 74 | + for (int i = streams.length - 1; i >= 0; i--) { |
| 75 | + streams[i].close(); |
97 | 76 | } |
98 | 77 | } |
99 | 78 |
|
100 | 79 | @Override |
101 | 80 | public void flush() throws IOException { |
102 | | - for (SingleFileCache cache : caches.values()) { |
103 | | - cache.flush(); |
104 | | - } |
| 81 | + recordConverter.flush(); |
105 | 82 | } |
106 | 83 |
|
| 84 | + /** |
| 85 | + * Compares time that the filecaches were last used. If equal, it lexicographically compares |
| 86 | + * the absolute path of the file. |
| 87 | + * @param other FileCache to compare with. |
| 88 | + */ |
107 | 89 | @Override |
108 | | - public void close() throws IOException { |
109 | | - try { |
110 | | - for (SingleFileCache cache : caches.values()) { |
111 | | - cache.close(); |
112 | | - } |
113 | | - } finally { |
114 | | - caches.clear(); |
| 90 | + public int compareTo(@Nonnull FileCache other) { |
| 91 | + int result = Long.compare(lastUse, other.lastUse); |
| 92 | + if (result != 0) { |
| 93 | + return result; |
115 | 94 | } |
| 95 | + return file.compareTo(other.file); |
116 | 96 | } |
117 | 97 |
|
118 | | - private class SingleFileCache implements Closeable, Flushable, Comparable<SingleFileCache> { |
119 | | - private final BufferedWriter bufferedWriter; |
120 | | - private final Writer fileWriter; |
121 | | - private final RecordConverter recordConverter; |
122 | | - private final File file; |
123 | | - private long lastUse; |
124 | | - |
125 | | - private SingleFileCache(File file, GenericRecord record) throws IOException { |
126 | | - this.file = file; |
127 | | - boolean fileIsNew = !file.exists() || file.length() == 0; |
128 | | - this.fileWriter = new FileWriter(file, true); |
129 | | - this.bufferedWriter = new BufferedWriter(fileWriter); |
130 | | - this.recordConverter = converterFactory.converterFor(bufferedWriter, record, fileIsNew); |
131 | | - } |
132 | | - |
133 | | - private void writeRecord(GenericRecord record) throws IOException { |
134 | | - this.recordConverter.writeRecord(record); |
135 | | - lastUse = System.nanoTime(); |
136 | | - } |
137 | | - |
138 | | - @Override |
139 | | - public void close() throws IOException { |
140 | | - recordConverter.close(); |
141 | | - bufferedWriter.close(); |
142 | | - fileWriter.close(); |
143 | | - } |
144 | | - |
145 | | - @Override |
146 | | - public void flush() throws IOException { |
147 | | - recordConverter.flush(); |
148 | | - } |
149 | | - |
150 | | - @Override |
151 | | - public int compareTo(@Nonnull SingleFileCache other) { |
152 | | - return Long.compare(lastUse, other.lastUse); |
153 | | - } |
154 | | - |
155 | | - private File getFile() { |
156 | | - return file; |
157 | | - } |
| 98 | + /** File that the cache is maintaining. */ |
| 99 | + public File getFile() { |
| 100 | + return file; |
158 | 101 | } |
159 | 102 | } |
0 commit comments