Skip to content

Commit fe71ef8

Browse files
adamretteradamretter
committed
[optimize] Reuse the CRC32 checksum object when compressing many files in the Compression Module
1 parent 542ea61 commit fe71ef8

File tree

1 file changed

+23
-16
lines changed

1 file changed

+23
-16
lines changed

‎extensions/modules/compression/src/main/java/org/exist/xquery/modules/compression/AbstractCompressFunction.java‎

Lines changed: 23 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -157,15 +157,17 @@ public Sequence eval(final Sequence[] args, final Sequence contextSequence) thro
157157
final OutputStream os = stream(baos, encoding);
158158
final StringBuilderWriter sbWriter = new StringBuilderWriter()) {
159159

160+
final CRC32 chksum = new CRC32();
161+
160162
// iterate through the argument sequence
161163
for (final SequenceIterator i = args[0].iterate(); i.hasNext(); ) {
162164
final Item item = i.nextItem();
163165

164166
if (item instanceof Element) {
165167
Element element = (Element) item;
166-
compressElement(os, element, useHierarchy, stripOffset, sbWriter);
168+
compressElement(os, element, useHierarchy, stripOffset, sbWriter, chksum);
167169
} else {
168-
compressFromUri(os, ((AnyURIValue) item).toURI(), useHierarchy, stripOffset, ZipMethod.DEFLATE, null, sbWriter);
170+
compressFromUri(os, ((AnyURIValue) item).toURI(), useHierarchy, stripOffset, ZipMethod.DEFLATE, null, sbWriter, chksum);
169171
}
170172
}
171173

@@ -182,7 +184,7 @@ public Sequence eval(final Sequence[] args, final Sequence contextSequence) thro
182184
}
183185
}
184186

185-
private void compressFromUri(final OutputStream os, final URI uri, final boolean useHierarchy, final String stripOffset, final ZipMethod method, final String resourceName, final StringBuilderWriter sbWriter) throws XPathException
187+
private void compressFromUri(final OutputStream os, final URI uri, final boolean useHierarchy, final String stripOffset, final ZipMethod method, final String resourceName, final StringBuilderWriter sbWriter, final CRC32 chksum) throws XPathException
186188
{
187189
try {
188190
if ("file".equals(uri.getScheme())) {
@@ -195,7 +197,7 @@ private void compressFromUri(final OutputStream os, final URI uri, final boolean
195197

196198
// got a file
197199
final Path file = Paths.get(uri.getPath());
198-
compressFile(os, file, useHierarchy, stripOffset, method, resourceName);
200+
compressFile(os, file, useHierarchy, stripOffset, method, resourceName, chksum);
199201

200202
} else {
201203

@@ -204,7 +206,7 @@ private void compressFromUri(final OutputStream os, final URI uri, final boolean
204206
// try for a collection
205207
try (final Collection collection = context.getBroker().openCollection(xmldburi, LockMode.READ_LOCK)) {
206208
if(collection != null) {
207-
compressCollection(os, collection, useHierarchy, stripOffset, method, sbWriter);
209+
compressCollection(os, collection, useHierarchy, stripOffset, method, sbWriter, chksum);
208210
return;
209211
}
210212
} catch (final PermissionDeniedException | LockException | SAXException | IOException pde) {
@@ -227,7 +229,7 @@ private void compressFromUri(final OutputStream os, final URI uri, final boolean
227229
throw new XPathException(this, "Invalid URI: " + uri.toString());
228230
}
229231

230-
compressResource(os, doc.getDocument(), useHierarchy, stripOffset, method, resourceName, sbWriter);
232+
compressResource(os, doc.getDocument(), useHierarchy, stripOffset, method, resourceName, sbWriter, chksum);
231233
return;
232234
}
233235
} catch (final PermissionDeniedException | LockException | SAXException | IOException pde) {
@@ -250,8 +252,9 @@ private void compressFromUri(final OutputStream os, final URI uri, final boolean
250252
* @param stripOffset a string that should be stripped from the start of the entry name.
251253
* @param method the Zip method.
252254
* @param name the name of the entry.
255+
* @param chksum an object that is used to calculate the checksum.
253256
*/
254-
private void compressFile(final OutputStream os, final Path file, final boolean useHierarchy, final String stripOffset, final ZipMethod method, final String name) throws IOException {
257+
private void compressFile(final OutputStream os, final Path file, final boolean useHierarchy, final String stripOffset, final ZipMethod method, final String name, final CRC32 chksum) throws IOException {
255258

256259
if (!Files.isDirectory(file)) {
257260

@@ -270,6 +273,7 @@ private void compressFile(final OutputStream os, final Path file, final boolean
270273
// close the entry
271274
if (entry instanceof ZipEntry && method == ZipMethod.STORE) {
272275
((ZipEntry) entry).setMethod(ZipOutputStream.STORED);
276+
chksum.reset();
273277
chksum.update(value);
274278
((ZipEntry) entry).setCrc(chksum.getValue());
275279
((ZipEntry) entry).setSize(value.length);
@@ -282,7 +286,7 @@ private void compressFile(final OutputStream os, final Path file, final boolean
282286
} else {
283287

284288
for (final Path child : FileUtils.list(file)) {
285-
compressFile(os, file.resolve(child), useHierarchy, stripOffset, method, null);
289+
compressFile(os, file.resolve(child), useHierarchy, stripOffset, method, null, chksum);
286290
}
287291

288292
}
@@ -297,9 +301,10 @@ private void compressFile(final OutputStream os, final Path file, final boolean
297301
* @param useHierarchy Whether to use a folder hierarchy in the archive file that reflects the collection hierarchy.
298302
* @param stripOffset a string that should be stripped from the start of the entry name.
299303
* @param sbWriter a StringBuilderWriter to reuse
304+
* @param chksum an object that is used to calculate the checksum.
300305
*/
301306
private void compressElement(final OutputStream os, final Element element, final boolean useHierarchy,
302-
final String stripOffset, final StringBuilderWriter sbWriter) throws XPathException {
307+
final String stripOffset, final StringBuilderWriter sbWriter, final CRC32 chksum) throws XPathException {
303308

304309
final String ns = element.getNamespaceURI();
305310
if (!(element.getNodeName().equals("entry") || (ns != null && !ns.isEmpty()))) {
@@ -327,7 +332,7 @@ private void compressElement(final OutputStream os, final Element element, final
327332
if (isNullOrEmpty(uri)) {
328333
throw new XPathException(this, "Entry with type uri must contain a URI.");
329334
}
330-
compressFromUri(os, URI.create(uri), useHierarchy, stripOffset, method, name, sbWriter);
335+
compressFromUri(os, URI.create(uri), useHierarchy, stripOffset, method, name, sbWriter, chksum);
331336
return;
332337
}
333338

@@ -347,7 +352,6 @@ private void compressElement(final OutputStream os, final Element element, final
347352

348353
if (!"collection".equals(type)) {
349354
final byte[] value;
350-
final CRC32 chksum = new CRC32();
351355
final Node content = element.getFirstChild();
352356

353357
if (content == null) {
@@ -380,6 +384,7 @@ private void compressElement(final OutputStream os, final Element element, final
380384

381385
if (entry instanceof ZipEntry && method == ZipMethod.STORE) {
382386
((ZipEntry) entry).setMethod(ZipOutputStream.STORED);
387+
chksum.reset();
383388
chksum.update(value);
384389
((ZipEntry) entry).setCrc(chksum.getValue());
385390
((ZipEntry) entry).setSize(value.length);
@@ -423,8 +428,9 @@ private void getDynamicSerializerOptions(final Serializer serializer) throws SAX
423428
* @param method the Zip method.
424429
* @param name the name of the entry.
425430
* @param sbWriter a StringBuilderWriter to reuse
431+
* @param chksum an object that is used to calculate the checksum.
426432
*/
427-
private void compressResource(final OutputStream os, final DocumentImpl doc, final boolean useHierarchy, final String stripOffset, final ZipMethod method, final String name, final StringBuilderWriter sbWriter) throws IOException, SAXException {
433+
private void compressResource(final OutputStream os, final DocumentImpl doc, final boolean useHierarchy, final String stripOffset, final ZipMethod method, final String name, final StringBuilderWriter sbWriter, final CRC32 chksum) throws IOException, SAXException {
428434
// create an entry in the Tar for the document
429435
final Object entry;
430436
if (name != null) {
@@ -464,9 +470,9 @@ private void compressResource(final OutputStream os, final DocumentImpl doc, fin
464470
}
465471

466472
// close the entry
467-
final CRC32 chksum = new CRC32();
468473
if (entry instanceof ZipEntry && method == ZipMethod.STORE) {
469474
((ZipEntry) entry).setMethod(ZipOutputStream.STORED);
475+
chksum.reset();
470476
chksum.update(value);
471477
((ZipEntry) entry).setCrc(chksum.getValue());
472478
((ZipEntry) entry).setSize(value.length);
@@ -486,8 +492,9 @@ private void compressResource(final OutputStream os, final DocumentImpl doc, fin
486492
* @param stripOffset a string that should be stripped from the start of the entry name.
487493
* @param method the Zip method.
488494
* @param sbWriter a StringBuilderWriter to reuse
495+
* @param chksum an object that is used to calculate the checksum.
489496
*/
490-
private void compressCollection(final OutputStream os, final Collection col, final boolean useHierarchy, final String stripOffset, final ZipMethod method, final StringBuilderWriter sbWriter) throws IOException, SAXException, LockException, PermissionDeniedException {
497+
private void compressCollection(final OutputStream os, final Collection col, final boolean useHierarchy, final String stripOffset, final ZipMethod method, final StringBuilderWriter sbWriter, final CRC32 chksum) throws IOException, SAXException, LockException, PermissionDeniedException {
491498
// iterate over child documents
492499
final DBBroker broker = context.getBroker();
493500
final LockManager lockManager = broker.getBrokerPool().getLockManager();
@@ -496,7 +503,7 @@ private void compressCollection(final OutputStream os, final Collection col, fin
496503
for (final Iterator<DocumentImpl> itChildDocs = childDocs.getDocumentIterator(); itChildDocs.hasNext();) {
497504
final DocumentImpl childDoc = itChildDocs.next();
498505
try (final ManagedDocumentLock updateLock = lockManager.acquireDocumentReadLock(childDoc.getURI())) {
499-
compressResource(os, childDoc, useHierarchy, stripOffset, method, null, sbWriter);
506+
compressResource(os, childDoc, useHierarchy, stripOffset, method, null, sbWriter, chksum);
500507
}
501508
}
502509
// iterate over child collections
@@ -505,7 +512,7 @@ private void compressCollection(final OutputStream os, final Collection col, fin
505512
final XmldbURI childColURI = itChildCols.next();
506513
final Collection childCol = broker.getCollection(col.getURI().append(childColURI));
507514
// recurse
508-
compressCollection(os, childCol, useHierarchy, stripOffset, method, sbWriter);
515+
compressCollection(os, childCol, useHierarchy, stripOffset, method, sbWriter, chksum);
509516
}
510517
}
511518

0 commit comments

Comments
 (0)