diff --git a/pom.xml b/pom.xml index 020f957..e6d152e 100644 --- a/pom.xml +++ b/pom.xml @@ -76,7 +76,7 @@ 3.25.3 1.78 - 1.26.1 + 1.28.0 1.16.1 33.1.0-jre 5.10.2 diff --git a/rpm/pom.xml b/rpm/pom.xml index b56f37b..e6b8998 100644 --- a/rpm/pom.xml +++ b/rpm/pom.xml @@ -36,21 +36,19 @@ org.tukaani xz - com.github.luben zstd-jni - 1.5.2-5 - true + 1.5.7-2 + test - --> org.testcontainers junit-jupiter test - ch.qos.logback logback-classic diff --git a/rpm/src/main/java/org/eclipse/packager/rpm/build/BuilderOptions.java b/rpm/src/main/java/org/eclipse/packager/rpm/build/BuilderOptions.java index 84e2dc0..c518ac8 100644 --- a/rpm/src/main/java/org/eclipse/packager/rpm/build/BuilderOptions.java +++ b/rpm/src/main/java/org/eclipse/packager/rpm/build/BuilderOptions.java @@ -24,6 +24,7 @@ import java.util.List; import org.eclipse.packager.rpm.coding.PayloadCoding; +import org.eclipse.packager.rpm.coding.PayloadFlags; /** * Options which control the build process of the {@link RpmBuilder} @@ -33,6 +34,11 @@ *

*/ public class BuilderOptions { + private static final PayloadCoding DEFAULT_PAYLOAD_CODING = PayloadCoding.GZIP; + + private static final PayloadFlags DEFAULT_PAYLOAD_FLAGS = new PayloadFlags(DEFAULT_PAYLOAD_CODING, 9); + + private LongMode longMode = LongMode.DEFAULT; private OpenOption[] openOptions; @@ -41,7 +47,7 @@ public class BuilderOptions { private PayloadCoding payloadCoding; - private String payloadFlags; + private PayloadFlags payloadFlags; private DigestAlgorithm fileDigestAlgorithm = DigestAlgorithm.MD5; @@ -98,18 +104,18 @@ public void setFileNameProvider(final RpmFileNameProvider fileNameProvider) { } public PayloadCoding getPayloadCoding() { - return this.payloadCoding != null ? this.payloadCoding : PayloadCoding.GZIP; + return this.payloadCoding != null ? this.payloadCoding : DEFAULT_PAYLOAD_CODING; } public void setPayloadCoding(final PayloadCoding payloadCoding) { this.payloadCoding = payloadCoding; } - public String getPayloadFlags() { - return this.payloadFlags; + public PayloadFlags getPayloadFlags() { + return (this.payloadFlags == null && this.payloadCoding == DEFAULT_PAYLOAD_CODING) ? DEFAULT_PAYLOAD_FLAGS : this.payloadFlags; } - public void setPayloadFlags(final String payloadFlags) { + public void setPayloadFlags(final PayloadFlags payloadFlags) { this.payloadFlags = payloadFlags; } diff --git a/rpm/src/main/java/org/eclipse/packager/rpm/build/PayloadProvider.java b/rpm/src/main/java/org/eclipse/packager/rpm/build/PayloadProvider.java index 10a6c53..66dc9e3 100644 --- a/rpm/src/main/java/org/eclipse/packager/rpm/build/PayloadProvider.java +++ b/rpm/src/main/java/org/eclipse/packager/rpm/build/PayloadProvider.java @@ -19,6 +19,7 @@ import org.eclipse.packager.rpm.RpmTag; import org.eclipse.packager.rpm.coding.PayloadCoding; +import org.eclipse.packager.rpm.coding.PayloadFlags; import org.eclipse.packager.rpm.header.Header; public interface PayloadProvider { @@ -61,7 +62,7 @@ public interface PayloadProvider { * * @return the compression flags for this compressed archive file, if any */ - Optional getPayloadFlags(); + PayloadFlags getPayloadFlags(); /** * The algorithm used for generating file checksum digests whose ordinal is diff --git a/rpm/src/main/java/org/eclipse/packager/rpm/build/PayloadRecorder.java b/rpm/src/main/java/org/eclipse/packager/rpm/build/PayloadRecorder.java index 2a067a6..aa32a44 100644 --- a/rpm/src/main/java/org/eclipse/packager/rpm/build/PayloadRecorder.java +++ b/rpm/src/main/java/org/eclipse/packager/rpm/build/PayloadRecorder.java @@ -39,15 +39,19 @@ import org.apache.commons.compress.archivers.cpio.CpioArchiveEntry; import org.apache.commons.compress.archivers.cpio.CpioArchiveOutputStream; import org.apache.commons.compress.archivers.cpio.CpioConstants; -import org.apache.commons.compress.utils.CharsetNames; import org.eclipse.packager.rpm.RpmTag; import org.eclipse.packager.rpm.coding.PayloadCoding; +import org.eclipse.packager.rpm.coding.PayloadFlags; import org.eclipse.packager.rpm.header.Header; import com.google.common.io.ByteStreams; import com.google.common.io.CountingOutputStream; public class PayloadRecorder implements AutoCloseable { + private static final PayloadCoding DEFAULT_PAYLOAD_CODING = PayloadCoding.GZIP; + + private static final PayloadFlags DEFAULT_PAYLOAD_FLAGS = new PayloadFlags(DEFAULT_PAYLOAD_CODING, 9); + public static class Result { private final long size; @@ -100,10 +104,10 @@ public void write(byte[] b, int off, int len) throws IOException { private Finished finished; public PayloadRecorder() throws IOException { - this(PayloadCoding.GZIP, null, DigestAlgorithm.MD5, null); + this(DEFAULT_PAYLOAD_CODING, DEFAULT_PAYLOAD_FLAGS, DigestAlgorithm.MD5, null); } - public PayloadRecorder(final PayloadCoding payloadCoding, final String payloadFlags, final DigestAlgorithm fileDigestAlgorithm, final List processors) throws IOException { + public PayloadRecorder(final PayloadCoding payloadCoding, final PayloadFlags payloadFlags, final DigestAlgorithm fileDigestAlgorithm, final List processors) throws IOException { this.fileDigestAlgorithm = fileDigestAlgorithm; if (processors == null) { this.processors = Collections.emptyList(); @@ -329,25 +333,24 @@ public class Finished implements AutoCloseable, PayloadProvider { private final PayloadCoding payloadCoding; - private final Optional payloadFlags; + private final PayloadFlags payloadFlags; private Header additionalHeader = new Header<>(); - private Finished(final PayloadCoding payloadCoding, final String payloadFlags) throws IOException { + private Finished(final PayloadCoding payloadCoding, final PayloadFlags payloadFlags) throws IOException { this.tempFile = Files.createTempFile("rpm-", null); try { final OutputStream fileStream = new BufferedOutputStream(Files.newOutputStream(this.tempFile, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING)); this.payloadCounter = new CountingOutputStream(new ProcessorStream(fileStream, PayloadRecorder.this::forEachCompressedData)); this.payloadCoding = payloadCoding; - this.payloadFlags = Optional.ofNullable(payloadFlags); - - final OutputStream payloadStream = new ProcessorStream(this.payloadCoding.createProvider().createOutputStream(this.payloadCounter, this.payloadFlags), PayloadRecorder.this::forEachRawData); + this.payloadFlags = payloadFlags; + final OutputStream payloadStream = new ProcessorStream(this.payloadCoding.createProvider().createOutputStream(this.payloadCounter, Optional.ofNullable(this.payloadFlags)), PayloadRecorder.this::forEachRawData); this.archiveCounter = new CountingOutputStream(payloadStream); // setup archive stream - this.archiveStream = new CpioArchiveOutputStream(this.archiveCounter, CpioConstants.FORMAT_NEW, 4, CharsetNames.UTF_8); + this.archiveStream = new CpioArchiveOutputStream(this.archiveCounter, CpioConstants.FORMAT_NEW, 4, StandardCharsets.UTF_8.name()); } catch (final IOException e) { Files.deleteIfExists(this.tempFile); throw e; @@ -376,7 +379,7 @@ public PayloadCoding getPayloadCoding() { } @Override - public Optional getPayloadFlags() { + public PayloadFlags getPayloadFlags() { return this.payloadFlags; } diff --git a/rpm/src/main/java/org/eclipse/packager/rpm/build/RpmBuilder.java b/rpm/src/main/java/org/eclipse/packager/rpm/build/RpmBuilder.java index 29e6f9e..bb7fea4 100644 --- a/rpm/src/main/java/org/eclipse/packager/rpm/build/RpmBuilder.java +++ b/rpm/src/main/java/org/eclipse/packager/rpm/build/RpmBuilder.java @@ -51,6 +51,8 @@ import org.eclipse.packager.rpm.Rpms; import org.eclipse.packager.rpm.VerifyFlags; import org.eclipse.packager.rpm.build.PayloadRecorder.Result; +import org.eclipse.packager.rpm.coding.PayloadCoding; +import org.eclipse.packager.rpm.coding.PayloadFlags; import org.eclipse.packager.rpm.deps.Dependencies; import org.eclipse.packager.rpm.deps.Dependency; import org.eclipse.packager.rpm.deps.RpmDependencyFlags; @@ -716,12 +718,16 @@ private void fillProvides() { private void fillHeader(final PayloadRecorder.Finished finished) { this.header.putString(RpmTag.PAYLOAD_FORMAT, "cpio"); - if (finished.getPayloadCoding() != null) { - this.header.putString(RpmTag.PAYLOAD_CODING, finished.getPayloadCoding().getValue()); + final PayloadCoding payloadCoding = finished.getPayloadCoding(); + + if (payloadCoding != PayloadCoding.NONE) { + this.header.putString(RpmTag.PAYLOAD_CODING, payloadCoding.toString()); } - if (finished.getPayloadFlags().isPresent()) { - this.header.putString(RpmTag.PAYLOAD_FLAGS, finished.getPayloadFlags().get()); + final PayloadFlags payloadFlags = finished.getPayloadFlags(); + + if (payloadFlags != null) { + this.header.putString(RpmTag.PAYLOAD_FLAGS, payloadFlags.toString()); } this.header.putStringArray(100, "C"); diff --git a/rpm/src/main/java/org/eclipse/packager/rpm/coding/BZip2PayloadCoding.java b/rpm/src/main/java/org/eclipse/packager/rpm/coding/BZip2PayloadCoding.java index 895a766..3235f8c 100644 --- a/rpm/src/main/java/org/eclipse/packager/rpm/coding/BZip2PayloadCoding.java +++ b/rpm/src/main/java/org/eclipse/packager/rpm/coding/BZip2PayloadCoding.java @@ -24,6 +24,10 @@ import org.eclipse.packager.rpm.deps.Dependency; import org.eclipse.packager.rpm.deps.RpmDependencyFlags; +import static org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream.MAX_BLOCKSIZE; +import static org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream.MIN_BLOCKSIZE; +import static org.eclipse.packager.rpm.coding.PayloadFlags.getLevel; + public class BZip2PayloadCoding implements PayloadCodingProvider { protected BZip2PayloadCoding() { } @@ -44,17 +48,11 @@ public InputStream createInputStream(final InputStream in) throws IOException { } @Override - public OutputStream createOutputStream(final OutputStream out, final Optional optionalFlags) throws IOException { - final String flags; - - final int blockSize; - - if (optionalFlags.isPresent() && (flags = optionalFlags.get()).length() > 0) { - blockSize = Integer.parseInt(flags.substring(0, 1)); - } else { - blockSize = BZip2CompressorOutputStream.MAX_BLOCKSIZE; - } + public OutputStream createOutputStream(final OutputStream out, final Optional optionalPayloadFlags) throws IOException { + final boolean smallMode = optionalPayloadFlags.map(PayloadFlags::getSmallMode).orElse(false); + final int level = getLevel(optionalPayloadFlags, MIN_BLOCKSIZE, MAX_BLOCKSIZE, MAX_BLOCKSIZE); + final int blockSize = (!smallMode || level <= 2) ? level : 2; return new BZip2CompressorOutputStream(out, blockSize); } } diff --git a/rpm/src/main/java/org/eclipse/packager/rpm/coding/DefaultPayloadCodingRegistry.java b/rpm/src/main/java/org/eclipse/packager/rpm/coding/DefaultPayloadCodingRegistry.java deleted file mode 100644 index b1ca0f6..0000000 --- a/rpm/src/main/java/org/eclipse/packager/rpm/coding/DefaultPayloadCodingRegistry.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright (c) 2015, 2019 Contributors to the Eclipse Foundation - * - * See the NOTICE file(s) distributed with this work for additional - * information regarding copyright ownership. - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Public License 2.0 which is available at - * http://www.eclipse.org/legal/epl-2.0 - * - * SPDX-License-Identifier: EPL-2.0 - */ - -package org.eclipse.packager.rpm.coding; - -import java.io.IOException; -import java.util.Map; -import java.util.TreeMap; - -public class DefaultPayloadCodingRegistry { - private static final String GZIP = "gzip"; - - private static final String BZIP2 = "bzip2"; - - private static final String LZMA = "lzma"; - - private static final String XZ = "xz"; - - private static final String ZSTD = "zstd"; - - private static final PayloadCodingProvider NULL_PAYLOAD_CODING = new NullPayloadCoding(); - - private static final Map REGISTRY = new TreeMap<>(); - - static { - REGISTRY.put(GZIP, new GzipPayloadCoding()); - REGISTRY.put(BZIP2, new BZip2PayloadCoding()); - REGISTRY.put(LZMA, new LZMAPayloadCoding()); - REGISTRY.put(XZ, new XZPayloadCoding()); - REGISTRY.put(ZSTD, new ZstdPayloadCoding()); - } - - public static PayloadCodingProvider get(final String coding) throws IOException { - if (coding == null) { - return NULL_PAYLOAD_CODING; - } - - final PayloadCodingProvider payloadCoding = REGISTRY.get(coding); - - if (payloadCoding == null) { - throw new IOException(String.format("Unknown payload coding '%s'", coding)); - } - - return payloadCoding; - } -} diff --git a/rpm/src/main/java/org/eclipse/packager/rpm/coding/GzipPayloadCoding.java b/rpm/src/main/java/org/eclipse/packager/rpm/coding/GzipPayloadCoding.java index eea9a9d..d164ac2 100644 --- a/rpm/src/main/java/org/eclipse/packager/rpm/coding/GzipPayloadCoding.java +++ b/rpm/src/main/java/org/eclipse/packager/rpm/coding/GzipPayloadCoding.java @@ -18,13 +18,16 @@ import java.io.OutputStream; import java.util.Optional; import java.util.function.Consumer; -import java.util.zip.Deflater; import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream; import org.apache.commons.compress.compressors.gzip.GzipParameters; import org.eclipse.packager.rpm.deps.Dependency; +import static java.util.zip.Deflater.BEST_COMPRESSION; +import static java.util.zip.Deflater.DEFAULT_COMPRESSION; +import static org.eclipse.packager.rpm.coding.PayloadFlags.getLevel; + public class GzipPayloadCoding implements PayloadCodingProvider { protected GzipPayloadCoding() { } @@ -44,20 +47,10 @@ public InputStream createInputStream(final InputStream in) throws IOException { } @Override - public OutputStream createOutputStream(final OutputStream out, final Optional optionalFlags) throws IOException { - final String flags; - final int compressionLevel; - - if (optionalFlags.isPresent() && (flags = optionalFlags.get()).length() > 0) { - compressionLevel = Integer.parseInt(flags.substring(0, 1)); - } else { - compressionLevel = Deflater.BEST_COMPRESSION; - } - + public OutputStream createOutputStream(final OutputStream out, final Optional optionalPayloadFlags) throws IOException { + final int level = getLevel(optionalPayloadFlags, DEFAULT_COMPRESSION, BEST_COMPRESSION, DEFAULT_COMPRESSION); final GzipParameters parameters = new GzipParameters(); - - parameters.setCompressionLevel(compressionLevel); - + parameters.setCompressionLevel(level); return new GzipCompressorOutputStream(out, parameters); } } diff --git a/rpm/src/main/java/org/eclipse/packager/rpm/coding/LZMAPayloadCoding.java b/rpm/src/main/java/org/eclipse/packager/rpm/coding/LZMAPayloadCoding.java index e497f90..d9b3308 100644 --- a/rpm/src/main/java/org/eclipse/packager/rpm/coding/LZMAPayloadCoding.java +++ b/rpm/src/main/java/org/eclipse/packager/rpm/coding/LZMAPayloadCoding.java @@ -23,6 +23,12 @@ import org.apache.commons.compress.compressors.lzma.LZMACompressorOutputStream; import org.eclipse.packager.rpm.deps.Dependency; import org.eclipse.packager.rpm.deps.RpmDependencyFlags; +import org.tukaani.xz.LZMA2Options; + +import static org.eclipse.packager.rpm.coding.PayloadFlags.getLevel; +import static org.tukaani.xz.LZMA2Options.PRESET_DEFAULT; +import static org.tukaani.xz.LZMA2Options.PRESET_MAX; +import static org.tukaani.xz.LZMA2Options.PRESET_MIN; public class LZMAPayloadCoding implements PayloadCodingProvider { protected LZMAPayloadCoding() { @@ -44,7 +50,8 @@ public InputStream createInputStream(final InputStream in) throws IOException { } @Override - public OutputStream createOutputStream(final OutputStream out, final Optional optionalFlags) throws IOException { - return new LZMACompressorOutputStream(out); + public OutputStream createOutputStream(final OutputStream out, final Optional optionalPayloadFlags) throws IOException { + final int preset = getLevel(optionalPayloadFlags, PRESET_MIN, PRESET_MAX, PRESET_DEFAULT); + return new LZMACompressorOutputStream.Builder().setOutputStream(out).setLzma2Options(new LZMA2Options(preset)).get(); } } diff --git a/rpm/src/main/java/org/eclipse/packager/rpm/coding/NullPayloadCoding.java b/rpm/src/main/java/org/eclipse/packager/rpm/coding/NullPayloadCoding.java index a83bb38..11474b7 100644 --- a/rpm/src/main/java/org/eclipse/packager/rpm/coding/NullPayloadCoding.java +++ b/rpm/src/main/java/org/eclipse/packager/rpm/coding/NullPayloadCoding.java @@ -13,7 +13,6 @@ package org.eclipse.packager.rpm.coding; -import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.util.Optional; @@ -32,15 +31,16 @@ public String getCoding() { @Override public void fillRequirements(final Consumer requirementsConsumer) { + } @Override - public InputStream createInputStream(final InputStream in) throws IOException { + public InputStream createInputStream(final InputStream in) { return in; } @Override - public OutputStream createOutputStream(final OutputStream out, final Optional optionalFlags) throws IOException { + public OutputStream createOutputStream(final OutputStream out, final Optional optionalPayloadFlags) { return out; } } diff --git a/rpm/src/main/java/org/eclipse/packager/rpm/coding/PayloadCoding.java b/rpm/src/main/java/org/eclipse/packager/rpm/coding/PayloadCoding.java index 253cf86..fbfd0c2 100644 --- a/rpm/src/main/java/org/eclipse/packager/rpm/coding/PayloadCoding.java +++ b/rpm/src/main/java/org/eclipse/packager/rpm/coding/PayloadCoding.java @@ -43,15 +43,20 @@ public PayloadCodingProvider createProvider() { public static Optional fromValue(final String payloadCoding) { if (payloadCoding == null) { - return Optional.of(GZIP); + return Optional.of(NONE); } for (final PayloadCoding coding : values()) { - if (coding.value.equals(payloadCoding)) { + if (coding.value.equalsIgnoreCase(payloadCoding)) { return Optional.of(coding); } } return Optional.empty(); } + + @Override + public String toString() { + return getValue(); + } } diff --git a/rpm/src/main/java/org/eclipse/packager/rpm/coding/PayloadCodingProvider.java b/rpm/src/main/java/org/eclipse/packager/rpm/coding/PayloadCodingProvider.java index eb7ae4c..6abce1d 100644 --- a/rpm/src/main/java/org/eclipse/packager/rpm/coding/PayloadCodingProvider.java +++ b/rpm/src/main/java/org/eclipse/packager/rpm/coding/PayloadCodingProvider.java @@ -36,6 +36,6 @@ default List getRequirements() { InputStream createInputStream(final InputStream in) throws IOException; - OutputStream createOutputStream(final OutputStream out, final Optional optionalFlags) throws IOException; + OutputStream createOutputStream(final OutputStream out, final Optional optionalPayloadFlags) throws IOException; } diff --git a/rpm/src/main/java/org/eclipse/packager/rpm/coding/PayloadFlags.java b/rpm/src/main/java/org/eclipse/packager/rpm/coding/PayloadFlags.java new file mode 100644 index 0000000..f7ae451 --- /dev/null +++ b/rpm/src/main/java/org/eclipse/packager/rpm/coding/PayloadFlags.java @@ -0,0 +1,365 @@ +/* + * Copyright (c) 2015, 2022 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ + +package org.eclipse.packager.rpm.coding; + +import java.util.Locale; +import java.util.Objects; +import java.util.Optional; +import java.util.zip.Deflater; + +import static org.apache.commons.compress.compressors.zstandard.ZstdConstants.ZSTD_WINDOWLOG_MAX; +import static org.apache.commons.compress.compressors.zstandard.ZstdConstants.ZSTD_WINDOWLOG_MIN; + +public class PayloadFlags { + private String coding; + + private Integer level; + + private String threads; + + private Integer strategy; + + private String windowLog; + + private boolean smallMode; + + public PayloadFlags() { + + } + + public PayloadFlags(final PayloadCoding payloadCoding) { + this.coding = payloadCoding.name(); + } + + public PayloadFlags(final PayloadCoding payloadCoding, final Integer level) { + this.coding = payloadCoding.name(); + this.level = level; + } + + public PayloadFlags(final PayloadCoding payloadCoding, final Integer level, final Integer strategy) { + this.coding = payloadCoding.name(); + this.level = level; + this.strategy = strategy; + } + + public PayloadFlags(final PayloadCoding payloadCoding, final Integer level, final boolean smallMode) { + this.coding = payloadCoding.name(); + this.level = level; + this.smallMode = smallMode; + } + + public PayloadFlags(final PayloadCoding payloadCoding, final Integer level, final String threads) { + this.coding = payloadCoding.name(); + this.level = level; + this.threads = threads; + } + + public PayloadFlags(final PayloadCoding payloadCoding, final Integer level, final String threads, final String windowLog) { + this.coding = payloadCoding.name(); + this.level = level; + this.threads = threads; + this.windowLog = windowLog; + } + + public PayloadFlags(final PayloadCoding payloadCoding, final String flags) { + this.coding = payloadCoding.name(); + + if (flags == null || flags.isEmpty()) { + return; + } + + switch (payloadCoding) { + case GZIP: + parseFlagsGzip(flags); + break; + case BZIP2: + parseFlagsBZip2(flags); + break; + case LZMA: + case XZ: + parseFlagsLZMA(flags); + break; + case ZSTD: + parseFlagsZstd(flags); + break; + case NONE: + break; + } + } + + public PayloadFlags(final String coding, final String flags) { + this(PayloadCoding.fromValue(coding).orElseThrow(() -> new IllegalArgumentException("Unknown payload coding '" + coding + "'")), flags); + } + + public static int getLevel(final Optional optionalPayloadFlags, final int lowestLevel, final int highestLevel, final int defaultLevel) { + if (optionalPayloadFlags.isEmpty()) { + return defaultLevel; + } + + final PayloadFlags payloadFlags = optionalPayloadFlags.get(); + + if (payloadFlags.getLevel() == null) { + return defaultLevel; + } + + if (payloadFlags.getLevel() < lowestLevel || payloadFlags.getLevel() > highestLevel) { + throw new IllegalArgumentException("Level " + payloadFlags.getLevel() + " must be between " + lowestLevel + " and " + highestLevel); + } + + return payloadFlags.getLevel(); + } + + public static int getThreads(final Optional optionalPayloadFlags) { + if (optionalPayloadFlags.isEmpty()) { + return 0; + } + + final PayloadFlags payloadFlags = optionalPayloadFlags.get(); + + if (payloadFlags.getThreads() == null) { + return 0; + } + + final int availableProcessors = Runtime.getRuntime().availableProcessors(); + + if (payloadFlags.getThreads().isEmpty()) { + return availableProcessors; + } + + final int threads = Integer.parseInt(payloadFlags.getThreads()); + + if (threads < 0) { + throw new IllegalArgumentException("Threads " + threads + " must be greater than or equal to 0"); + } + + if (threads == 0) { + return availableProcessors; + } + + return threads; + } + + public static int getWindowLog(final Optional optionalPayloadFlags) { + if (optionalPayloadFlags.isEmpty()) { + return 0; + } + + final PayloadFlags payloadFlags = optionalPayloadFlags.get(); + + if (payloadFlags.getWindowLog() == null || payloadFlags.getWindowLog().isEmpty()) { + return 0; + } + + final int windowLog = Integer.parseInt(payloadFlags.getWindowLog()); + + if (windowLog < ZSTD_WINDOWLOG_MIN || windowLog > ZSTD_WINDOWLOG_MAX) { + throw new IllegalArgumentException("Window log " + windowLog + " must be between " + ZSTD_WINDOWLOG_MIN + " and " + ZSTD_WINDOWLOG_MAX); + } + + return windowLog; + } + + public String getCoding() { + return this.coding; + } + + public void setCoding(final String coding) { + this.coding = coding != null ? coding.toLowerCase(Locale.ROOT) : null; + } + + public Integer getLevel() { + return this.level; + } + + public void setLevel(final Integer level) { + this.level = level; + } + + public String getThreads() { + return this.threads; + } + + public void setThreads(final String threads) { + this.threads = threads; + } + + public Integer getStrategy() { + return this.strategy; + } + + public void setStrategy(final Integer strategy) { + this.strategy = strategy; + } + + public String getWindowLog() { + return this.windowLog; + } + + public void setWindowLog(final String windowLog) { + this.windowLog = windowLog; + } + + public boolean getSmallMode() { + return this.smallMode; + } + + public void setSmallMode(final boolean smallMode) { + this.smallMode = smallMode; + } + + private void parseFlagsGzip(final String flags) { + int i = 0; + + while (i < flags.length()) { + final char c = flags.charAt(i); + + if (Character.isDigit(c)) { + this.level = Integer.parseInt(String.valueOf(c)); + } else if (c == 'f') { + this.strategy = Deflater.FILTERED; + } else if (c == 'h') { + this.strategy = Deflater.HUFFMAN_ONLY; + } + + i++; + } + } + + private void parseFlagsBZip2(final String flags) { + int i = 0; + + while (i < flags.length()) { + final char c = flags.charAt(i); + + if (Character.isDigit(c)) { + level = Integer.parseInt(String.valueOf(c)); + } else if (c == 's') { + smallMode = true; + } + + i++; + } + } + + private void parseFlagsLZMA(final String flags) { + int i = 0; + + while (i < flags.length()) { + final char c = flags.charAt(i); + + if (Character.isDigit(c)) { + level = Integer.parseInt(String.valueOf(c)); + i++; + } else if (c == 'T') { + final int start = ++i; + + while (i < flags.length() && Character.isDigit(flags.charAt(i))) { + i++; + } + + threads = flags.substring(start, i); + } + } + } + + private void parseFlagsZstd(final String flags) { + int i = 0; + + while (i < flags.length()) { + final char c = flags.charAt(i); + + if (Character.isDigit(c)) { + final int start = i; + + while (i < flags.length() && Character.isDigit(flags.charAt(i))) { + i++; + } + + level = Integer.parseInt(flags.substring(start, i)); + } else if (c == 'L') { + final int start = ++i; + + while (i < flags.length() && Character.isDigit(flags.charAt(i))) { + i++; + } + + windowLog = flags.substring(start, i); + } else if (c == 'T') { + final int start = ++i; + + while (i < flags.length() && Character.isDigit(flags.charAt(i))) { + i++; + } + + threads = flags.substring(start, i); + } + } + } + + @Override + public boolean equals(final Object o) { + if (o == null || getClass() != o.getClass()) { + return false; + } + + final PayloadFlags that = (PayloadFlags) o; + return Objects.equals(this.coding, that.coding) && Objects.equals(this.level, that.level) && Objects.equals(this.threads, that.threads) && Objects.equals(this.strategy, that.strategy) && Objects.equals(this.windowLog, that.windowLog) && Objects.equals(this.smallMode, that.smallMode); + } + + @Override + public int hashCode() { + return Objects.hash(this.coding, level, this.threads, this.strategy, this.windowLog, this.smallMode); + } + + @Override + public String toString() { + if (this.coding == null) { + return ""; + } + + final PayloadCoding payloadCoding = PayloadCoding.valueOf(this.coding); + + if (payloadCoding == PayloadCoding.NONE) { + return ""; + } + + final StringBuilder sb = new StringBuilder(); + + if (this.level != null) { + sb.append(this.level); + } + + if (this.strategy != null && payloadCoding == PayloadCoding.BZIP2) { + if (this.strategy == Deflater.FILTERED) { + sb.append('f'); + } else if (this.strategy == Deflater.HUFFMAN_ONLY) { + sb.append('h'); + } + } + + if (this.smallMode && payloadCoding == PayloadCoding.BZIP2) { + sb.append('s'); + } + + if (this.threads != null && payloadCoding != PayloadCoding.GZIP && payloadCoding != PayloadCoding.BZIP2) { + sb.append('T').append(this.threads); + } + + if (this.windowLog != null && payloadCoding == PayloadCoding.ZSTD) { + sb.append('L').append(windowLog); + } + + return sb.toString(); + } +} diff --git a/rpm/src/main/java/org/eclipse/packager/rpm/coding/XZPayloadCoding.java b/rpm/src/main/java/org/eclipse/packager/rpm/coding/XZPayloadCoding.java index d4d51a9..914b9e1 100644 --- a/rpm/src/main/java/org/eclipse/packager/rpm/coding/XZPayloadCoding.java +++ b/rpm/src/main/java/org/eclipse/packager/rpm/coding/XZPayloadCoding.java @@ -25,6 +25,11 @@ import org.eclipse.packager.rpm.deps.RpmDependencyFlags; import org.tukaani.xz.LZMA2Options; +import static org.eclipse.packager.rpm.coding.PayloadFlags.getLevel; +import static org.tukaani.xz.LZMA2Options.PRESET_DEFAULT; +import static org.tukaani.xz.LZMA2Options.PRESET_MAX; +import static org.tukaani.xz.LZMA2Options.PRESET_MIN; + public class XZPayloadCoding implements PayloadCodingProvider { protected XZPayloadCoding() { } @@ -45,16 +50,8 @@ public InputStream createInputStream(final InputStream in) throws IOException { } @Override - public OutputStream createOutputStream(final OutputStream out, final Optional optionalFlags) throws IOException { - final String flags; - final int preset; - - if (optionalFlags.isPresent() && (flags = optionalFlags.get()).length() > 0) { - preset = Integer.parseInt(flags.substring(0, 1)); - } else { - preset = LZMA2Options.PRESET_DEFAULT; - } - - return new XZCompressorOutputStream(out, preset); + public OutputStream createOutputStream(final OutputStream out, final Optional optionalPayloadFlags) throws IOException { + final int preset = getLevel(optionalPayloadFlags, PRESET_MIN, PRESET_MAX, PRESET_DEFAULT); + return new XZCompressorOutputStream.Builder().setOutputStream(out).setLzma2Options(new LZMA2Options(preset)).get(); } } diff --git a/rpm/src/main/java/org/eclipse/packager/rpm/coding/ZstdPayloadCoding.java b/rpm/src/main/java/org/eclipse/packager/rpm/coding/ZstdPayloadCoding.java index a6eb870..df0712e 100644 --- a/rpm/src/main/java/org/eclipse/packager/rpm/coding/ZstdPayloadCoding.java +++ b/rpm/src/main/java/org/eclipse/packager/rpm/coding/ZstdPayloadCoding.java @@ -16,6 +16,8 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; import java.util.Optional; import java.util.function.Consumer; @@ -25,7 +27,37 @@ import org.eclipse.packager.rpm.deps.Dependency; import org.eclipse.packager.rpm.deps.RpmDependencyFlags; +import static java.lang.Integer.MAX_VALUE; +import static java.lang.Integer.MIN_VALUE; +import static org.eclipse.packager.rpm.coding.PayloadFlags.getLevel; +import static org.eclipse.packager.rpm.coding.PayloadFlags.getThreads; +import static org.eclipse.packager.rpm.coding.PayloadFlags.getWindowLog; + public class ZstdPayloadCoding implements PayloadCodingProvider { + public static final boolean ZSTD_COMPRESSION_AVAILABLE = ZstdUtils.isZstdCompressionAvailable(); + + public static Integer MIN_COMPRESSION_LEVEL = MIN_VALUE; + + public static Integer MAX_COMPRESSION_LEVEL = MAX_VALUE; + + public static Integer DEFAULT_COMPRESSION_LEVEL = 0; + + static { + if (ZSTD_COMPRESSION_AVAILABLE) { + try { + final Class zstdClass = Class.forName("com.github.luben.zstd.Zstd"); + final Method minCompressionLevelMethod = zstdClass.getMethod("minCompressionLevel"); + MIN_COMPRESSION_LEVEL = (Integer) minCompressionLevelMethod.invoke(null); + final Method maxCompressionLevelMethod = zstdClass.getMethod("maxCompressionLevel"); + MAX_COMPRESSION_LEVEL = (Integer) maxCompressionLevelMethod.invoke(null); + final Method defaultCompressionLevelMethod = zstdClass.getMethod("defaultCompressionLevel"); + DEFAULT_COMPRESSION_LEVEL = (Integer) defaultCompressionLevelMethod.invoke(null); + } catch (ClassNotFoundException | NoSuchMethodException | IllegalAccessException | InvocationTargetException e) { + throw new RuntimeException("Failed to initialize Zstd compression levels", e); + } + } + } + protected ZstdPayloadCoding() { } @@ -41,7 +73,7 @@ public void fillRequirements(final Consumer requirementsConsumer) { @Override public InputStream createInputStream(final InputStream in) throws IOException { - if (!ZstdUtils.isZstdCompressionAvailable()) { + if (!ZSTD_COMPRESSION_AVAILABLE) { throw new IOException("Zstandard compression is not available"); } @@ -49,21 +81,14 @@ public InputStream createInputStream(final InputStream in) throws IOException { } @Override - public OutputStream createOutputStream(final OutputStream out, final Optional optionalFlags) throws IOException { - if (!ZstdUtils.isZstdCompressionAvailable()) { + public OutputStream createOutputStream(final OutputStream out, final Optional optionalPayloadFlags) throws IOException { + if (!ZSTD_COMPRESSION_AVAILABLE) { throw new IOException("Zstandard compression is not available"); } - final String flags; - - final int level; - - if (optionalFlags.isPresent() && (flags = optionalFlags.get()).length() > 0) { - level = Integer.parseInt(flags.substring(0, 1)); - } else { - level = 3; - } - - return new ZstdCompressorOutputStream(out, level); + final int level = getLevel(optionalPayloadFlags, MIN_COMPRESSION_LEVEL, MAX_COMPRESSION_LEVEL, DEFAULT_COMPRESSION_LEVEL); + final int workers = getThreads(optionalPayloadFlags); + final int windowLog = getWindowLog(optionalPayloadFlags); + return new ZstdCompressorOutputStream.Builder().setOutputStream(out).setLevel(level).setWorkers(workers).setWindowLog(windowLog).get(); } } diff --git a/rpm/src/main/java/org/eclipse/packager/rpm/parse/RpmInputStream.java b/rpm/src/main/java/org/eclipse/packager/rpm/parse/RpmInputStream.java index 0c5a958..e87ad15 100644 --- a/rpm/src/main/java/org/eclipse/packager/rpm/parse/RpmInputStream.java +++ b/rpm/src/main/java/org/eclipse/packager/rpm/parse/RpmInputStream.java @@ -95,18 +95,13 @@ private InputStream setupPayloadStream() throws IOException { } if (!"cpio".equals(payloadFormat)) { - throw new IOException(String.format("Unknown payload format: %s", payloadFormat)); + throw new IOException("Unknown payload format '" + payloadFormat + "' , expected 'cpio'"); } // payload coding final String payloadCoding = this.payloadHeader.getString(RpmTag.PAYLOAD_CODING); - - if (payloadCoding == null) { - throw new IOException("Payload coding must be a single string"); - } - - final PayloadCoding coding = PayloadCoding.fromValue(payloadCoding).orElseThrow(() -> new IOException(String.format("Unknown payload coding: '%s'", payloadCoding))); + final PayloadCoding coding = PayloadCoding.fromValue(payloadCoding).orElseThrow(() -> new IOException("Unknown payload coding '" + payloadCoding + "'")); return coding.createProvider().createInputStream(this.in); } diff --git a/rpm/src/test/java/org/eclipse/packager/rpm/PayloadCodingTest.java b/rpm/src/test/java/org/eclipse/packager/rpm/PayloadCodingTest.java new file mode 100644 index 0000000..81162ea --- /dev/null +++ b/rpm/src/test/java/org/eclipse/packager/rpm/PayloadCodingTest.java @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2015, 2019 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ + +package org.eclipse.packager.rpm; + +import org.apache.commons.compress.compressors.zstandard.ZstdUtils; +import org.eclipse.packager.rpm.app.Dumper; +import org.eclipse.packager.rpm.build.BuilderOptions; +import org.eclipse.packager.rpm.build.RpmBuilder; +import org.eclipse.packager.rpm.coding.PayloadCoding; +import org.eclipse.packager.rpm.coding.PayloadFlags; +import org.eclipse.packager.rpm.parse.RpmInputStream; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.EnumSource; + +import java.io.BufferedInputStream; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.assertj.core.api.Assertions.assertThat; + +class PayloadCodingTest { + @BeforeAll + static void init() { + // Ensure that Zstd compression is available for the test + assertThat(ZstdUtils.isZstdCompressionAvailable()).isTrue(); + } + + @TempDir + private Path outBase; + + @ParameterizedTest + @EnumSource + void testPayloadCoding(final PayloadCoding payloadCoding) throws IOException { + final BuilderOptions options = new BuilderOptions(); + options.setPayloadCoding(payloadCoding); + final PayloadFlags payloadFlags = new PayloadFlags(payloadCoding, 9); + options.setPayloadFlags(payloadFlags); + + try (final RpmBuilder builder = new RpmBuilder(payloadCoding.toString(), RpmVersion.valueOf("1.0.0-1"), "noarch", outBase, options)) { + final Path outFile = builder.getTargetFile(); + builder.build(); + + try (final RpmInputStream in = new RpmInputStream(new BufferedInputStream(Files.newInputStream(outFile)))) { + Dumper.dumpAll(in); + final String payloadCodingString = in.getPayloadHeader().getString(RpmTag.PAYLOAD_CODING); + + if (payloadCodingString != null) { + assertThat(payloadCodingString).isEqualTo(payloadCoding.toString()); + } + + final String payloadFlagsString = in.getPayloadHeader().getString(RpmTag.PAYLOAD_FLAGS); + + if (payloadCoding == PayloadCoding.NONE) { + assertThat(payloadFlagsString).isEmpty(); + } else { + assertThat(payloadFlagsString).isEqualTo("9"); + } + } + } + } + + @ParameterizedTest + @CsvSource({"gzip,9", "bzip2,9", "xz,6", "xz,7T16", "xz,7T0", "xz,7T", "lzma,6", "zstd,3", "zstd,19T8", "zstd,7T0", "none,", "zstd,7L", "zstd,7L0"}) + void testPayloadFlags(final String payloadCoding, final String payloadFlagsString) { + final PayloadFlags payloadFlags = new PayloadFlags(payloadCoding, payloadFlagsString); + + if (payloadFlagsString != null) { + assertThat(payloadFlags).hasToString(payloadFlagsString); + } else { + assertThat(payloadFlags.toString()).isEmpty(); + } + } +}