Skip to content

Commit 07e7383

Browse files
PKJoshPKJosh
committed
Draft commit for reducing read IOPS in temporary package parts.
1 parent 7fbc3a4 commit 07e7383

File tree

6 files changed

+480
-3
lines changed

6 files changed

+480
-3
lines changed

‎poi-ooxml/src/main/java/org/apache/poi/openxml4j/opc/ZipPackage.java‎

Lines changed: 25 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,6 @@ Licensed to the Apache Software Foundation (ASF) under one or more
5252
import org.apache.poi.openxml4j.util.ZipFileZipEntrySource;
5353
import org.apache.poi.openxml4j.util.ZipInputStreamZipEntrySource;
5454
import org.apache.poi.util.IOUtils;
55-
import org.apache.poi.util.TempFile;
5655

5756
/**
5857
* Physical zip package.
@@ -62,6 +61,7 @@ public final class ZipPackage extends OPCPackage {
6261
private static final String SETTINGS_XML = "settings.xml";
6362
private static boolean useTempFilePackageParts = false;
6463
private static boolean encryptTempFilePackageParts = false;
64+
private static boolean bufferTempFilePackageParts = false;
6565

6666
private static final Logger LOG = LogManager.getLogger(ZipPackage.class);
6767

@@ -85,6 +85,13 @@ public static void setEncryptTempFilePackageParts(boolean encryptTempFiles) {
8585
encryptTempFilePackageParts = encryptTempFiles;
8686
}
8787

88+
/**
89+
* @param bufferTempFiles whether to buffer package part temp files
90+
*/
91+
public static void setBufferTempFilePackageParts(boolean bufferTempFiles) {
92+
bufferTempFilePackageParts = bufferTempFiles;
93+
}
94+
8895
/**
8996
* @return whether package part data is stored in temp files to save memory
9097
*/
@@ -99,6 +106,13 @@ public static boolean encryptTempFilePackageParts() {
99106
return encryptTempFilePackageParts;
100107
}
101108

109+
/**
110+
* @return whether package part temp files are biffered
111+
*/
112+
public static boolean bufferTempFilePackageParts() {
113+
return bufferTempFilePackageParts;
114+
}
115+
102116
/**
103117
* Constructor. Creates a new, empty ZipPackage.
104118
*/
@@ -422,8 +436,16 @@ protected PackagePart createPartImpl(PackagePartName partName,
422436
try {
423437
if (useTempFilePackageParts) {
424438
if (encryptTempFilePackageParts) {
425-
return new EncryptedTempFilePackagePart(this, partName, contentType, loadRelationships);
426-
} else {
439+
if (bufferTempFilePackageParts) {
440+
return new BufferedEncryptedTempFilePackagePart(this, partName, contentType, loadRelationships);
441+
} else {
442+
return new EncryptedTempFilePackagePart(this, partName, contentType, loadRelationships);
443+
}
444+
}
445+
else if (bufferTempFilePackageParts) {
446+
return new BufferedTempFilePackagePart(this, partName, contentType, loadRelationships);
447+
}
448+
else {
427449
return new TempFilePackagePart(this, partName, contentType, loadRelationships);
428450
}
429451
} else {

‎poi-ooxml/src/main/java/org/apache/poi/openxml4j/opc/internal/BufferedEncryptedTempFilePackagePart.java‎

Lines changed: 147 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
/* ====================================================================
2+
Licensed to the Apache Software Foundation (ASF) under one or more
3+
contributor license agreements. See the NOTICE file distributed with
4+
this work for additional information regarding copyright ownership.
5+
The ASF licenses this file to You under the Apache License, Version 2.0
6+
(the "License"); you may not use this file except in compliance with
7+
the License. You may obtain a copy of the License at
8+
9+
http://www.apache.org/licenses/LICENSE-2.0
10+
11+
Unless required by applicable law or agreed to in writing, software
12+
distributed under the License is distributed on an "AS IS" BASIS,
13+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
See the License for the specific language governing permissions and
15+
limitations under the License.
16+
==================================================================== */
17+
18+
package org.apache.poi.openxml4j.opc.internal;
19+
20+
import org.apache.logging.log4j.LogManager;
21+
import org.apache.logging.log4j.Logger;
22+
import org.apache.poi.openxml4j.exceptions.InvalidFormatException;
23+
import org.apache.poi.openxml4j.exceptions.OpenXML4JException;
24+
import org.apache.poi.openxml4j.opc.OPCPackage;
25+
import org.apache.poi.openxml4j.opc.PackagePart;
26+
import org.apache.poi.openxml4j.opc.PackagePartName;
27+
import org.apache.poi.openxml4j.opc.internal.marshallers.ZipPartMarshaller;
28+
import org.apache.poi.poifs.crypt.temp.EncryptedTempData;
29+
import org.apache.poi.util.Beta;
30+
import org.apache.poi.util.IOUtils;
31+
32+
import java.io.BufferedInputStream;
33+
import java.io.BufferedOutputStream;
34+
import java.io.FileInputStream;
35+
import java.io.FileOutputStream;
36+
import java.io.IOException;
37+
import java.io.InputStream;
38+
import java.io.OutputStream;
39+
40+
/**
41+
* (Experimental) Buffered Encrypted Temp File version of a package part.
42+
*
43+
* @since POI 5.1.0
44+
*/
45+
@Beta
46+
public final class BufferedEncryptedTempFilePackagePart extends PackagePart {
47+
private static int fileBufferSize = 1024 * 1024; // 1 MB
48+
private static final Logger LOG = LogManager.getLogger(BufferedEncryptedTempFilePackagePart.class);
49+
50+
/**
51+
* Storage for the part data.
52+
*/
53+
private EncryptedTempData tempFile;
54+
55+
/**
56+
* Constructor.
57+
*
58+
* @param pack
59+
* The owner package.
60+
* @param partName
61+
* The part name.
62+
* @param contentType
63+
* The content type.
64+
* @throws InvalidFormatException
65+
* If the specified URI is not OPC compliant.
66+
* @throws IOException
67+
* If temp file cannot be created.
68+
*/
69+
public BufferedEncryptedTempFilePackagePart(OPCPackage pack, PackagePartName partName,
70+
String contentType) throws InvalidFormatException, IOException {
71+
this(pack, partName, contentType, true);
72+
}
73+
74+
/**
75+
* Constructor.
76+
*
77+
* @param pack
78+
* The owner package.
79+
* @param partName
80+
* The part name.
81+
* @param contentType
82+
* The content type.
83+
* @param loadRelationships
84+
* Specify if the relationships will be loaded.
85+
* @throws InvalidFormatException
86+
* If the specified URI is not OPC compliant.
87+
* @throws IOException
88+
* If temp file cannot be created.
89+
*/
90+
public BufferedEncryptedTempFilePackagePart(OPCPackage pack, PackagePartName partName,
91+
String contentType, boolean loadRelationships)
92+
throws InvalidFormatException, IOException {
93+
super(pack, partName, new ContentType(contentType), loadRelationships);
94+
tempFile = new EncryptedTempData();
95+
}
96+
97+
@Override
98+
protected InputStream getInputStreamImpl() throws IOException {
99+
return new BufferedInputStream(tempFile.getInputStream(), fileBufferSize);
100+
}
101+
102+
@Override
103+
protected OutputStream getOutputStreamImpl() throws IOException {
104+
return new BufferedOutputStream(tempFile.getOutputStream(), fileBufferSize);
105+
}
106+
107+
@Override
108+
public long getSize() {
109+
return tempFile.getByteCount();
110+
}
111+
112+
@Override
113+
public void clear() {
114+
try(OutputStream os = getOutputStreamImpl()) {
115+
os.write(new byte[0]);
116+
} catch (IOException e) {
117+
LOG.atWarn().log("Failed to clear data in temp file", e);
118+
}
119+
}
120+
121+
@Override
122+
public boolean save(OutputStream os) throws OpenXML4JException {
123+
return new ZipPartMarshaller().marshall(this, os);
124+
}
125+
126+
@Override
127+
public boolean load(InputStream is) throws InvalidFormatException {
128+
try (OutputStream os = getOutputStreamImpl()) {
129+
IOUtils.copy(is, os);
130+
} catch(IOException e) {
131+
throw new InvalidFormatException(e.getMessage(), e);
132+
}
133+
134+
// All done
135+
return true;
136+
}
137+
138+
@Override
139+
public void close() {
140+
tempFile.dispose();
141+
}
142+
143+
@Override
144+
public void flush() {
145+
// Do nothing
146+
}
147+
}

‎poi-ooxml/src/main/java/org/apache/poi/openxml4j/opc/internal/BufferedTempFilePackagePart.java‎

Lines changed: 159 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,159 @@
1+
/* ====================================================================
2+
Licensed to the Apache Software Foundation (ASF) under one or more
3+
contributor license agreements. See the NOTICE file distributed with
4+
this work for additional information regarding copyright ownership.
5+
The ASF licenses this file to You under the Apache License, Version 2.0
6+
(the "License"); you may not use this file except in compliance with
7+
the License. You may obtain a copy of the License at
8+
9+
http://www.apache.org/licenses/LICENSE-2.0
10+
11+
Unless required by applicable law or agreed to in writing, software
12+
distributed under the License is distributed on an "AS IS" BASIS,
13+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
See the License for the specific language governing permissions and
15+
limitations under the License.
16+
==================================================================== */
17+
18+
package org.apache.poi.openxml4j.opc.internal;
19+
20+
import org.apache.logging.log4j.LogManager;
21+
import org.apache.logging.log4j.Logger;
22+
import org.apache.poi.openxml4j.exceptions.InvalidFormatException;
23+
import org.apache.poi.openxml4j.exceptions.OpenXML4JException;
24+
import org.apache.poi.openxml4j.opc.OPCPackage;
25+
import org.apache.poi.openxml4j.opc.PackagePart;
26+
import org.apache.poi.openxml4j.opc.PackagePartName;
27+
import org.apache.poi.openxml4j.opc.internal.marshallers.ZipPartMarshaller;
28+
import org.apache.poi.util.Beta;
29+
import org.apache.poi.util.IOUtils;
30+
import org.apache.poi.util.TempFile;
31+
32+
import java.io.BufferedInputStream;
33+
import java.io.BufferedOutputStream;
34+
import java.io.File;
35+
import java.io.FileInputStream;
36+
import java.io.FileOutputStream;
37+
import java.io.IOException;
38+
import java.io.InputStream;
39+
import java.io.OutputStream;
40+
41+
/**
42+
* (Experimental) Buffered Temp File version of a package part.
43+
*
44+
* @since POI 5.4.2
45+
*/
46+
@Beta
47+
public final class BufferedTempFilePackagePart extends PackagePart {
48+
private static int fileBufferSize = 1024 * 1024; // 1 MB
49+
private static final Logger LOG = LogManager.getLogger(BufferedTempFilePackagePart.class);
50+
51+
/**
52+
* Storage for the part data.
53+
*/
54+
private File tempFile;
55+
56+
/**
57+
* Constructor.
58+
*
59+
* @param pack
60+
* The owner package.
61+
* @param partName
62+
* The part name.
63+
* @param contentType
64+
* The content type.
65+
* @throws InvalidFormatException
66+
* If the specified URI is not OPC compliant.
67+
* @throws IOException
68+
* If temp file cannot be created.
69+
*/
70+
public BufferedTempFilePackagePart(OPCPackage pack, PackagePartName partName,
71+
String contentType) throws InvalidFormatException, IOException {
72+
this(pack, partName, contentType, true);
73+
}
74+
75+
/**
76+
* Constructor.
77+
*
78+
* @param pack
79+
* The owner package.
80+
* @param partName
81+
* The part name.
82+
* @param contentType
83+
* The content type.
84+
* @param loadRelationships
85+
* Specify if the relationships will be loaded.
86+
* @throws InvalidFormatException
87+
* If the specified URI is not OPC compliant.
88+
* @throws IOException
89+
* If temp file cannot be created.
90+
*/
91+
public BufferedTempFilePackagePart(OPCPackage pack, PackagePartName partName,
92+
String contentType, boolean loadRelationships)
93+
throws InvalidFormatException, IOException {
94+
super(pack, partName, new ContentType(contentType), loadRelationships);
95+
tempFile = TempFile.createTempFile("poi-package-part", ".tmp");
96+
}
97+
98+
/**
99+
*
100+
* @param bufferSize
101+
* Size by whic
102+
*/
103+
public static void setBufferSize(int bufferSize) {
104+
fileBufferSize = bufferSize;
105+
}
106+
107+
@Override
108+
protected InputStream getInputStreamImpl() throws IOException {
109+
return new BufferedInputStream(new FileInputStream(tempFile), fileBufferSize);
110+
}
111+
112+
@Override
113+
protected OutputStream getOutputStreamImpl() throws IOException {
114+
return new BufferedOutputStream(new FileOutputStream(tempFile), fileBufferSize);
115+
}
116+
117+
@Override
118+
public long getSize() {
119+
return tempFile.length();
120+
}
121+
122+
@Override
123+
public void clear() {
124+
try(OutputStream os = getOutputStreamImpl()) {
125+
os.write(new byte[0]);
126+
} catch (IOException e) {
127+
LOG.atWarn().log("Failed to clear data in temp file", e);
128+
}
129+
}
130+
131+
@Override
132+
public boolean save(OutputStream os) throws OpenXML4JException {
133+
return new ZipPartMarshaller().marshall(this, os);
134+
}
135+
136+
@Override
137+
public boolean load(InputStream is) throws InvalidFormatException {
138+
try (OutputStream os = getOutputStreamImpl()) {
139+
IOUtils.copy(is, os);
140+
} catch(IOException e) {
141+
throw new InvalidFormatException(e.getMessage(), e);
142+
}
143+
144+
// All done
145+
return true;
146+
}
147+
148+
@Override
149+
public void close() {
150+
if (!tempFile.delete()) {
151+
LOG.atInfo().log("Failed to delete temp file; may already have been closed and deleted");
152+
}
153+
}
154+
155+
@Override
156+
public void flush() {
157+
// Do nothing
158+
}
159+
}

0 commit comments

Comments
 (0)