Add support for decompressing bz2, gz, xz, zst

Bazel already supports decompressing tar files compressed with bz2, gz, xz, and zst. This change adds support for decompressing individual files (not in a tar bundle) that are compressed with those compression algorithms.

In other words, support for decompression:

before: .tar.bz2, .tar.gz, .tar.xz, .tar.zst
after : .tar.bz2, .bz2, .tar.gz, .gz, .tar.xz, .xz, .tar.zst, .zst

Fixes: #20125

Closes #27413.

PiperOrigin-RevId: 839740791
Change-Id: I82595436f6ecab23374db7c50a5027a4bb279578

Author: willstranton

src/main/java/com/google/devtools/build/lib/bazel/repository/decompressor/Bz2Function.java

@@ -0,0 +1,38 @@
+// Copyright 2025 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.bazel.repository.decompressor;
+
+import com.google.devtools.build.lib.bazel.repository.decompressor.DecompressorValue.Decompressor;
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
+import org.apache.commons.compress.compressors.bzip2.BZip2Utils;
+
+/** Decompresses a bzip2 compressed file. */
+public class Bz2Function extends CompressedFunction {
+  public static final Decompressor INSTANCE = new Bz2Function();
+
+  @Override
+  protected InputStream getDecompressorStream(BufferedInputStream compressedInputStream)
+      throws IOException {
+    return new BZip2CompressorInputStream(compressedInputStream, true);
+  }
+
+  @Override
+  protected String getUncompressedFileName(InputStream in, String compressedFileName) {
+    return BZip2Utils.getUncompressedFileName(compressedFileName);
+  }
+}

src/main/java/com/google/devtools/build/lib/bazel/repository/decompressor/CompressedFunction.java

@@ -0,0 +1,83 @@
+// Copyright 2025 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.bazel.repository.decompressor;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.lib.bazel.repository.decompressor.DecompressorValue.Decompressor;
+import com.google.devtools.build.lib.vfs.Path;
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+/**
+ * Common code for decompressing a single compressed file (compressor formats).
+ *
+ * <p>Apache Commons Compress calls all formats that compress a single stream of data compressor
+ * formats while all formats that collect multiple entries inside a single (potentially compressed)
+ * archive are archiver formats. This class handles the former, compressor formats.
+ *
+ * <p>It ignores the {@link DecompressorDescriptor#prefix()} setting because compressed files cannot
+ * contain directories.
+ */
+public abstract class CompressedFunction implements Decompressor {
+
+  protected abstract InputStream getDecompressorStream(BufferedInputStream compressedInputStream)
+      throws IOException;
+
+  /**
+   * Returns the uncompressed file name (eg. file.gz -> file). Some compressors have metadata that
+   * stores the original name. If that's the case, the original name is used (eg. file.gz ->
+   * originalName). Only a basename + ext should be passed in for the compressedFileName.
+   */
+  protected abstract String getUncompressedFileName(
+      InputStream in, final String compressedFileName);
+
+  /**
+   * Set custom file attributes, like last modified time, on the extracted file. Only certain
+   * compressors support this.
+   */
+  protected void setFileAttributes(InputStream in, Path uncompressedFile) throws IOException {}
+
+  // This is the same value as picked for .tar files, which appears to have worked well.
+  private static final int BUFFER_SIZE = 32 * 1024;
+
+  @Override
+  public Path decompress(DecompressorDescriptor descriptor)
+      throws InterruptedException, IOException {
+    if (Thread.interrupted()) {
+      throw new InterruptedException();
+    }
+
+    ImmutableMap<String, String> renameFiles = descriptor.renameFiles();
+    try (InputStream decompressorStream =
+        getDecompressorStream(
+            new BufferedInputStream(descriptor.archivePath().getInputStream(), BUFFER_SIZE))) {
+      String entryName =
+          getUncompressedFileName(decompressorStream, descriptor.archivePath().getBaseName());
+      entryName = renameFiles.getOrDefault(entryName, entryName);
+      Path filePath = descriptor.destinationPath().getRelative(entryName);
+      filePath.getParentDirectory().createDirectoryAndParents();
+      try (OutputStream out = filePath.getOutputStream()) {
+        decompressorStream.transferTo(out);
+      }
+      setFileAttributes(decompressorStream, filePath);
+      if (Thread.interrupted()) {
+        throw new InterruptedException();
+      }
+    }
+    return descriptor.destinationPath();
+  }
+}

src/main/java/com/google/devtools/build/lib/bazel/repository/decompressor/DecompressorValue.java

@@ -104,12 +104,20 @@ static Decompressor getDecompressor(Path archivePath) throws RepositoryFunctionE
       return TarFunction.INSTANCE;
     } else if (baseName.endsWith(".tar.gz") || baseName.endsWith(".tgz")) {
       return TarGzFunction.INSTANCE;
+    } else if (baseName.endsWith(".gz")) { // Must be after .tar.gz.
+      return GzFunction.INSTANCE;
     } else if (baseName.endsWith(".tar.xz") || baseName.endsWith(".txz")) {
       return TarXzFunction.INSTANCE;
+    } else if (baseName.endsWith(".xz")) { // Must be after .tar.xz.
+      return XzFunction.INSTANCE;
     } else if (baseName.endsWith(".tar.zst") || baseName.endsWith(".tzst")) {
       return TarZstFunction.INSTANCE;
+    } else if (baseName.endsWith(".zst")) { // Must be after .tar.zst.
+      return ZstFunction.INSTANCE;
     } else if (baseName.endsWith(".tar.bz2") || baseName.endsWith(".tbz")) {
       return TarBz2Function.INSTANCE;
+    } else if (baseName.endsWith(".bz2")) { // Must be after .tar.bz2.
+      return Bz2Function.INSTANCE;
     } else if (baseName.endsWith(".ar") || baseName.endsWith(".deb")) {
       return ArFunction.INSTANCE;
     } else if (baseName.endsWith(".7z")) {
@@ -118,7 +126,8 @@ static Decompressor getDecompressor(Path archivePath) throws RepositoryFunctionE
       throw new RepositoryFunctionException(
           Starlark.errorf(
               "Expected a file with a .zip, .jar, .war, .aar, .nupkg, .whl, .tar, .tar.gz, .tgz,"
-                  + " .tar.xz, , .tar.zst, .tzst, .tar.bz2, .tbz, .ar, .deb or .7z suffix (got %s)",
+                  + " .gz, .tar.xz, .txz, .xz, .tar.zst, .tzst, .zst, .tar.bz2, .tbz, .bz2, .ar,"
+                  + " .deb or .7z suffix (got %s)",
               archivePath),
           Transience.PERSISTENT);
     }

src/main/java/com/google/devtools/build/lib/bazel/repository/decompressor/GzFunction.java

@@ -0,0 +1,53 @@
+// Copyright 2025 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.bazel.repository.decompressor;
+
+import com.google.devtools.build.lib.bazel.repository.decompressor.DecompressorValue.Decompressor;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
+import org.apache.commons.compress.compressors.gzip.GzipParameters;
+import org.apache.commons.compress.compressors.gzip.GzipUtils;
+
+/** Decompresses a gzip compressed file. */
+public class GzFunction extends CompressedFunction {
+  public static final Decompressor INSTANCE = new GzFunction();
+
+  @Override
+  protected InputStream getDecompressorStream(BufferedInputStream compressedInputStream)
+      throws IOException {
+    return new GzipCompressorInputStream(compressedInputStream, true);
+  }
+
+  @Override
+  protected String getUncompressedFileName(InputStream in, String compressedFileName) {
+    String fileName = ((GzipCompressorInputStream) in).getMetaData().getFileName();
+    if (fileName != null && !fileName.isBlank()) {
+      // filename should be the simple basename + ext, but convert to a PathFragment and run
+      // getBaseName to ensure that any path separators and uplevel references are dropped.
+      return PathFragment.create(fileName).getBaseName();
+    }
+    return GzipUtils.getUncompressedFileName(compressedFileName);
+  }
+
+  @Override
+  protected void setFileAttributes(InputStream in, Path uncompressedFile) throws IOException {
+    GzipParameters metaData = ((GzipCompressorInputStream) in).getMetaData();
+    uncompressedFile.setLastModifiedTime(metaData.getModificationTime() * 1000);
+  }
+}

src/main/java/com/google/devtools/build/lib/bazel/repository/decompressor/XzFunction.java

@@ -0,0 +1,50 @@
+// Copyright 2025 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.bazel.repository.decompressor;
+
+import com.google.devtools.build.lib.bazel.repository.decompressor.DecompressorValue.Decompressor;
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import org.apache.commons.compress.compressors.xz.XZCompressorInputStream;
+import org.apache.commons.compress.compressors.xz.XZUtils;
+
+/** Decompresses an xz (LZMA) compressed file. */
+public class XzFunction extends CompressedFunction {
+  public static final Decompressor INSTANCE = new XzFunction();
+
+  /**
+   * Uses {@link XZCompressorInputStream} from Apache Commons Compress to decompress.
+   *
+   * <p>Why not use {@link org.tukaani.xz.XZInputStream} which is used in {@link TarXzFunction}? The
+   * Apache Commons Compress libraries are wrappers around org.tukaani.xz.XZInputStream, so they
+   * should be the same. Since we also use {@link
+   * org.apache.commons.compress.compressors.xz.XZUtils}, we keep consistency and use the Apache
+   * wrapper consistently in this class.
+   *
+   * @see <a
+   *     href="https://commons.apache.org/proper/commons-compress/apidocs/org/apache/commons/compress/compressors/xz/package-summary.html">javadoc</a>
+   */
+  @Override
+  protected InputStream getDecompressorStream(BufferedInputStream compressedInputStream)
+      throws IOException {
+    return new XZCompressorInputStream(compressedInputStream);
+  }
+
+  @Override
+  protected String getUncompressedFileName(InputStream in, String compressedFileName) {
+    return XZUtils.getUncompressedFileName(compressedFileName);
+  }
+}

src/main/java/com/google/devtools/build/lib/bazel/repository/decompressor/ZstFunction.java

@@ -0,0 +1,42 @@
+// Copyright 2025 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.bazel.repository.decompressor;
+
+import com.github.luben.zstd.ZstdInputStreamNoFinalizer;
+import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.lib.bazel.repository.decompressor.DecompressorValue.Decompressor;
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import org.apache.commons.compress.compressors.FileNameUtil;
+
+/** Decompresses a Zstandard compressed file. */
+public class ZstFunction extends CompressedFunction {
+  public static final Decompressor INSTANCE = new ZstFunction();
+  // Apache Commons Compress does not provide a readily available mapping of compressed ->
+  // uncompressed filenames for Zst, so we make our own.
+  static final FileNameUtil fileNameUtil = new FileNameUtil(ImmutableMap.of(".zst", ""), ".zst");
+
+  @Override
+  protected InputStream getDecompressorStream(BufferedInputStream compressedInputStream)
+      throws IOException {
+    return new ZstdInputStreamNoFinalizer(compressedInputStream);
+  }
+
+  @Override
+  protected String getUncompressedFileName(InputStream in, String compressedFileName) {
+    return fileNameUtil.getUncompressedFileName(compressedFileName);
+  }
+}

src/main/java/com/google/devtools/build/lib/bazel/repository/starlark/StarlarkBaseExternalContext.java

@@ -912,8 +912,8 @@ public Object download(
                 The archive type of the downloaded file. By default, the archive type is \
                 determined from the file extension of the URL. If the file has no \
                 extension, you can explicitly specify either "zip", "jar", "war", \
-                "aar", "nupkg", "whl", "tar", "tar.gz", "tgz", "tar.xz", "txz", ".tar.zst", \
-                ".tzst", "tar.bz2", ".tbz", ".ar", ".deb", or ".7z" here.
+                "aar", "nupkg", "whl", "tar", "tar.gz", "tgz", "gz", "tar.xz", "txz", "xz", "tar.zst", \
+                "tzst", "zst", "tar.bz2", "tbz", "bz2", "ar", "deb", or "7z" here.
                 """),
         @Param(
             name = "strip_prefix",

src/test/java/com/google/devtools/build/lib/bazel/repository/decompressor/BUILD

@@ -40,6 +40,7 @@ java_library(
         "//third_party:junit4",
         "//third_party:truth",
         "@rules_java//java/runfiles",
+        "@zstd-jni",
     ],
 )