Skip to content

Commit fac92b5

Browse files
msimacekansalond
authored andcommitted
[GR-68825] Backport to 25.0: Fix truncated decompression in java zlib backend
PullRequest: graalpython/3963
2 parents 7add173 + 387d77d commit fac92b5

File tree

10 files changed

+83
-16
lines changed

10 files changed

+83
-16
lines changed

graalpython/com.oracle.graal.python.shell/src/com/oracle/graal/python/shell/GraalPythonMain.java

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -182,6 +182,7 @@ protected List<String> preprocessArguments(List<String> givenArgs, Map<String, S
182182
origArgs = new ArrayList<>();
183183
boolean posixBackendSpecified = false;
184184
boolean sha3BackendSpecified = false;
185+
boolean compressionBackendSpecified = false;
185186
boolean installSignalHandlersSpecified = false;
186187
boolean isolateNativeModulesSpecified = false;
187188
for (Iterator<String> argumentIterator = arguments.iterator(); argumentIterator.hasNext();) {
@@ -271,7 +272,8 @@ protected List<String> preprocessArguments(List<String> givenArgs, Map<String, S
271272
matchesPythonOption(arg, "StdLibHome") ||
272273
matchesPythonOption(arg, "CAPI") ||
273274
matchesPythonOption(arg, "PosixModuleBackend") ||
274-
matchesPythonOption(arg, "Sha3ModuleBackend")) {
275+
matchesPythonOption(arg, "Sha3ModuleBackend") ||
276+
matchesPythonOption(arg, "CompressionModulesBackend")) {
275277
addRelaunchArg(arg);
276278
}
277279
if (matchesPythonOption(arg, "PosixModuleBackend")) {
@@ -280,6 +282,9 @@ protected List<String> preprocessArguments(List<String> givenArgs, Map<String, S
280282
if (matchesPythonOption(arg, "Sha3ModuleBackend")) {
281283
sha3BackendSpecified = true;
282284
}
285+
if (matchesPythonOption(arg, "CompressionModulesBackend")) {
286+
compressionBackendSpecified = true;
287+
}
283288
if (matchesPythonOption(arg, "InstallSignalHandlers")) {
284289
installSignalHandlersSpecified = true;
285290
}
@@ -444,6 +449,9 @@ protected List<String> preprocessArguments(List<String> givenArgs, Map<String, S
444449
if (!sha3BackendSpecified) {
445450
polyglotOptions.put("python.Sha3ModuleBackend", "native");
446451
}
452+
if (!compressionBackendSpecified) {
453+
polyglotOptions.put("python.CompressionModulesBackend", "native");
454+
}
447455
if (!installSignalHandlersSpecified) {
448456
polyglotOptions.put("python.InstallSignalHandlers", "true");
449457
}

graalpython/com.oracle.graal.python.test/src/tests/test_zlib.py

Lines changed: 35 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,11 +14,12 @@ def _disable_native_zlib(self, flag):
1414
return None
1515
__graalpython__ = GP()
1616

17+
import binascii
1718
import os
19+
import random
20+
import sys
1821
import unittest
1922
import zlib
20-
import binascii
21-
import sys
2223

2324
pintNumber = 98765432109876543210
2425
longNumber = 9876543210
@@ -271,3 +272,35 @@ def test_GR65704():
271272
__graalpython__._disable_native_zlib(False)
272273

273274
assert decompressed == contents
275+
276+
def test_large_chunk():
277+
contents = random.randbytes(5000)
278+
wbits = 31
279+
280+
__graalpython__._disable_native_zlib(True)
281+
282+
compressed = zlib.compress(contents, wbits=wbits)
283+
decompressor = zlib.decompressobj(wbits=wbits)
284+
285+
decompressed = decompressor.decompress(compressed)
286+
287+
__graalpython__._disable_native_zlib(False)
288+
289+
assert decompressed == contents
290+
291+
def test_various_chunks():
292+
contents = random.randbytes(5000)
293+
wbits = 31
294+
295+
__graalpython__._disable_native_zlib(True)
296+
297+
compressed = zlib.compress(contents, wbits=wbits)
298+
decompressor = zlib.decompressobj(wbits=wbits)
299+
300+
decompressed = decompressor.decompress(compressed[:10])
301+
decompressed += decompressor.decompress(compressed[10:200])
302+
decompressed += decompressor.decompress(compressed[200:])
303+
304+
__graalpython__._disable_native_zlib(False)
305+
306+
assert decompressed == contents

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/zlib/JavaDecompress.java

Lines changed: 21 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@
4747

4848
import java.io.ByteArrayInputStream;
4949
import java.io.ByteArrayOutputStream;
50+
import java.io.EOFException;
5051
import java.io.IOException;
5152
import java.io.InputStream;
5253
import java.util.zip.DataFormatException;
@@ -112,7 +113,7 @@ public Inflater getInflater() {
112113
return inf;
113114
}
114115

115-
public void setInput() throws IOException {
116+
public void fillInput() throws IOException {
116117
fill();
117118
}
118119
}
@@ -162,7 +163,7 @@ private static boolean isGZIPStreamReady(DecompressStream stream, byte[] data, i
162163
// GZIPInputStream will read the header during initialization
163164
stream.stream = new GZIPDecompressStream(stream.in);
164165
stream.inflater = stream.stream.getInflater();
165-
stream.stream.setInput();
166+
stream.stream.fillInput();
166167
return true;
167168
}
168169
} catch (ZipException ze) {
@@ -178,7 +179,7 @@ private static boolean isGZIPStreamFinishing(DecompressStream stream, byte[] dat
178179
stream.in.append(data, 0, length);
179180
try {
180181
if (stream.in.length() >= HEADER_TRAILER_SIZE) {
181-
stream.stream.setInput();
182+
stream.stream.fillInput();
182183
// this should trigger reading trailer
183184
stream.stream.read();
184185
stream.stream = null;
@@ -243,22 +244,33 @@ private byte[] createByteArray(byte[] bytes, int length, int maxLength, int bufS
243244
return EMPTY_BYTE_ARRAY;
244245
}
245246

246-
int maxLen = maxLength == 0 ? Integer.MAX_VALUE : maxLength;
247+
int maxLen = maxLength <= 0 ? Integer.MAX_VALUE : maxLength;
247248
byte[] result = new byte[Math.min(maxLen, bufSize)];
248249

249-
int bytesWritten = result.length;
250250
ByteArrayOutputStream baos = new ByteArrayOutputStream();
251251
boolean zdictIsSet = false;
252-
while (baos.size() < maxLen && bytesWritten == result.length) {
252+
while (baos.size() < maxLen && !stream.inflater.finished()) {
253+
if (stream.inflater.needsInput()) {
254+
if (stream.stream == null) {
255+
break;
256+
}
257+
try {
258+
stream.stream.fillInput();
259+
} catch (EOFException e) {
260+
break;
261+
} catch (IOException e) {
262+
throw CompilerDirectives.shouldNotReachHere(e);
263+
}
264+
}
265+
int bytesWritten;
253266
try {
254267
int len = Math.min(maxLen - baos.size(), result.length);
255268
bytesWritten = stream.inflater.inflate(result, 0, len);
256269
if (bytesWritten == 0 && !zdictIsSet && stream.inflater.needsDictionary()) {
257270
if (getZdict().length > 0) {
258271
setDictionary();
259272
zdictIsSet = true;
260-
// we inflate again with a dictionary
261-
bytesWritten = stream.inflater.inflate(result, 0, len);
273+
continue;
262274
} else {
263275
throw PRaiseNode.raiseStatic(nodeForRaise, ZLibError, WHILE_SETTING_ZDICT);
264276
}
@@ -320,7 +332,7 @@ protected static byte[] decompress(byte[] bytes, int length, int wbits, int bufs
320332
private void saveUnconsumedInput(byte[] data, int length,
321333
byte[] unusedDataBytes, int unconsumedTailLen, Node inliningTarget) {
322334
int unusedLen = getRemaining();
323-
byte[] tail = PythonUtils.arrayCopyOfRange(data, length - unusedLen, length);
335+
byte[] tail = PythonUtils.arrayCopyOfRange(data, Math.max(0, length - unusedLen), length);
324336
PythonLanguage language = PythonLanguage.get(inliningTarget);
325337
if (isEof()) {
326338
if (unconsumedTailLen > 0) {

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/NFIBz2Support.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -153,7 +153,7 @@ public String signature() {
153153
@CompilerDirectives.CompilationFinal private boolean available;
154154

155155
private NFIBz2Support(PythonContext context, NativeLibrary.NFIBackend backend, String noNativeAccessHelp) {
156-
if (context.isNativeAccessAllowed()) {
156+
if (context.useNativeCompressionModules()) {
157157
this.pythonContext = context;
158158
this.typedNativeLib = NativeLibrary.create(PythonContext.getSupportLibName(SUPPORTING_NATIVE_LIB_NAME), Bz2NativeFunctions.values(),
159159
backend, noNativeAccessHelp, false);

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/NFILZMASupport.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -293,7 +293,7 @@ public String signature() {
293293
@CompilerDirectives.CompilationFinal private boolean available;
294294

295295
private NFILZMASupport(PythonContext context, NativeLibrary.NFIBackend backend, String noNativeAccessHelp) {
296-
if (context.isNativeAccessAllowed()) {
296+
if (context.useNativeCompressionModules()) {
297297
this.pythonContext = context;
298298
this.typedNativeLib = NativeLibrary.create(PythonContext.getSupportLibName(SUPPORTING_NATIVE_LIB_NAME), LZMANativeFunctions.values(),
299299
backend, noNativeAccessHelp, true);

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/NFIZlibSupport.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -269,7 +269,7 @@ public String signature() {
269269
@CompilerDirectives.CompilationFinal private boolean available;
270270

271271
private NFIZlibSupport(PythonContext context, NativeLibrary.NFIBackend backend, String noNativeAccessHelp) {
272-
if (context.isNativeAccessAllowed()) {
272+
if (context.useNativeCompressionModules()) {
273273
this.pythonContext = context;
274274
this.typedNativeLib = NativeLibrary.create(PythonContext.getSupportLibName(SUPPORTING_NATIVE_LIB_NAME),
275275
ZlibNativeFunctions.values(), backend, noNativeAccessHelp, true);

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonContext.java

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2335,6 +2335,15 @@ public boolean isExecutableAccessAllowed() {
23352335
return getEnv().isHostLookupAllowed() || isNativeAccessAllowed();
23362336
}
23372337

2338+
public boolean useNativeCompressionModules() {
2339+
if (isNativeAccessAllowed()) {
2340+
TruffleString option = getLanguage().getEngineOption(PythonOptions.CompressionModulesBackend);
2341+
TruffleString.EqualNode eqNode = TruffleString.EqualNode.getUncached();
2342+
return !eqNode.execute(T_JAVA, option, TS_ENCODING);
2343+
}
2344+
return false;
2345+
}
2346+
23382347
/**
23392348
* Trigger any pending asynchronous actions
23402349
*/

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonOptions.java

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -195,12 +195,15 @@ public static void checkBytecodeDSLEnv() {
195195
}
196196
}));
197197

198-
@EngineOption @Option(category = OptionCategory.USER, help = "Choose the backend for the POSIX module.", usageSyntax = "java|native|llvm", stability = OptionStability.STABLE) //
198+
@EngineOption @Option(category = OptionCategory.USER, help = "Choose the backend for the POSIX module.", usageSyntax = "java|native", stability = OptionStability.STABLE) //
199199
public static final OptionKey<TruffleString> PosixModuleBackend = new OptionKey<>(T_JAVA, TS_OPTION_TYPE);
200200

201201
@EngineOption @Option(category = OptionCategory.USER, help = "Choose the backend for the Sha3 module.", usageSyntax = "java|native", stability = OptionStability.STABLE) //
202202
public static final OptionKey<TruffleString> Sha3ModuleBackend = new OptionKey<>(T_JAVA, TS_OPTION_TYPE);
203203

204+
@EngineOption @Option(category = OptionCategory.USER, help = "Choose the backend for the Zlib, Bz2, and LZMA modules.", usageSyntax = "java|native", stability = OptionStability.STABLE) //
205+
public static final OptionKey<TruffleString> CompressionModulesBackend = new OptionKey<>(T_JAVA, TS_OPTION_TYPE);
206+
204207
@Option(category = OptionCategory.USER, help = "Install default signal handlers on startup", usageSyntax = "true|false", stability = OptionStability.STABLE) //
205208
public static final OptionKey<Boolean> InstallSignalHandlers = new OptionKey<>(false);
206209

mx.graalpython/mx_graalpython.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1925,6 +1925,7 @@ def bytecode_dsl_build_args():
19251925
'--enable-native-access=org.graalvm.shadowed.jline',
19261926
'-Dpolyglot.python.PosixModuleBackend=native',
19271927
'-Dpolyglot.python.Sha3ModuleBackend=native',
1928+
'-Dpolyglot.python.CompressionModulesBackend=native',
19281929
] + bytecode_dsl_build_args(),
19291930
language='python',
19301931
default_vm_args=[

mx.graalpython/suite.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -810,6 +810,7 @@
810810
'--enable-native-access=org.graalvm.shadowed.jline',
811811
"-Dpolyglot.python.PosixModuleBackend=native",
812812
"-Dpolyglot.python.Sha3ModuleBackend=native",
813+
"-Dpolyglot.python.CompressionModulesBackend=native",
813814
],
814815
"dynamicBuildArgs": "libpythonvm_build_args",
815816
},

0 commit comments

Comments
 (0)