Support zip permissions and symlinks with Apache Commons Compress

build.mill

@@ -26,6 +26,7 @@ object Deps {
   val sourcecode = ivy"com.lihaoyi::sourcecode::0.4.2"
   val utest = ivy"com.lihaoyi::utest::0.8.4"
   val expecty = ivy"com.eed3si9n.expecty::expecty::0.16.0"
+  val apacheCommonsCompress = ivy"org.apache.commons:commons-compress:1.27.1"
   def scalaReflect(scalaVersion: String) = ivy"org.scala-lang:scala-reflect:$scalaVersion"
   def scalaLibrary(version: String) = ivy"org.scala-lang:scala-library:${version}"
 }
@@ -112,7 +113,7 @@ trait OsLibModule
 }
 
 trait OsModule extends OsLibModule { outer =>
-  def ivyDeps = Agg(Deps.geny)
+  def ivyDeps = Agg(Deps.geny, Deps.apacheCommonsCompress)
   override def compileIvyDeps = T {
     val scalaReflectOpt = Option.when(!ZincWorkerUtil.isDottyOrScala3(scalaVersion()))(
       Deps.scalaReflect(scalaVersion())

os/src/PermissionUtils.java

@@ -0,0 +1,153 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one or more
+ *  contributor license agreements.  See the NOTICE file distributed with
+ *  this work for additional information regarding copyright ownership.
+ *  The ASF licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+
+// taken from https://github.com/apache/ant/blob/ecfca50b0133c576021dd088f855ee878a6b8e66/src/main/org/apache/tools/ant/util/PermissionUtils.java
+package os;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.nio.file.attribute.PosixFileAttributeView;
+import java.nio.file.attribute.PosixFilePermission;
+import java.util.EnumSet;
+import java.util.Set;
+
+/**
+ * Contains helper methods for dealing with {@link
+ * PosixFilePermission} or the traditional Unix mode representation of
+ * permissions.
+ */
+class PermissionUtils {
+
+    private PermissionUtils() { }
+
+    /**
+     * Translates a set of permissions into a Unix stat(2) {@code
+     * st_mode} result.
+     * @param permissions the permissions
+     * @param type the file type
+     * @return the "mode"
+     */
+    public static int modeFromPermissions(Set<PosixFilePermission> permissions,
+                                          FileType type) {
+        int mode;
+        switch (type) {
+        case SYMLINK:
+            mode = 012;
+            break;
+        case REGULAR_FILE:
+            mode = 010;
+            break;
+        case DIR:
+            mode = 004;
+            break;
+        default:
+            // OTHER could be a character or block device, a socket or a FIFO - so don't set anything
+            mode = 0;
+            break;
+        }
+        mode <<= 3;
+        mode <<= 3; // we don't support sticky, setuid, setgid
+        mode |= modeFromPermissions(permissions, "OWNER");
+        mode <<= 3;
+        mode |= modeFromPermissions(permissions, "GROUP");
+        mode <<= 3;
+        mode |= modeFromPermissions(permissions, "OTHERS");
+        return mode;
+    }
+
+    /**
+     * Translates a Unix stat(2) {@code st_mode} compatible value into
+     * a set of permissions.
+     * @param mode the "mode"
+     * @return set of permissions
+     */
+    public static Set<PosixFilePermission> permissionsFromMode(int mode) {
+        Set<PosixFilePermission> permissions = EnumSet.noneOf(PosixFilePermission.class);
+        addPermissions(permissions, "OTHERS", mode);
+        addPermissions(permissions, "GROUP", mode >> 3);
+        addPermissions(permissions, "OWNER", mode >> 6);
+        return permissions;
+    }
+
+    private static long modeFromPermissions(Set<PosixFilePermission> permissions,
+                                            String prefix) {
+        long mode = 0;
+        if (permissions.contains(PosixFilePermission.valueOf(prefix + "_READ"))) {
+            mode |= 4;
+        }
+        if (permissions.contains(PosixFilePermission.valueOf(prefix + "_WRITE"))) {
+            mode |= 2;
+        }
+        if (permissions.contains(PosixFilePermission.valueOf(prefix + "_EXECUTE"))) {
+            mode |= 1;
+        }
+        return mode;
+    }
+
+    private static void addPermissions(Set<PosixFilePermission> permissions,
+                                       String prefix, long mode) {
+        if ((mode & 1) == 1) {
+            permissions.add(PosixFilePermission.valueOf(prefix + "_EXECUTE"));
+        }
+        if ((mode & 2) == 2) {
+            permissions.add(PosixFilePermission.valueOf(prefix + "_WRITE"));
+        }
+        if ((mode & 4) == 4) {
+            permissions.add(PosixFilePermission.valueOf(prefix + "_READ"));
+        }
+    }
+
+    /**
+     * The supported types of files, maps to the {@code isFoo} methods
+     * in {@link java.nio.file.attribute.BasicFileAttributes}.
+     */
+    public enum FileType {
+        /** A regular file. */
+        REGULAR_FILE,
+        /** A directory. */
+        DIR,
+        /** A symbolic link. */
+        SYMLINK,
+        /** Something that is neither a regular file nor a directory nor a symbolic link. */
+        OTHER;
+
+        /**
+         * Determines the file type of a {@link Path}.
+         *
+         * @param p Path
+         * @return FileType
+         * @throws IOException if file attributes cannot be read
+         */
+        public static FileType of(Path p) throws IOException {
+            BasicFileAttributes attrs =
+                Files.readAttributes(p, BasicFileAttributes.class);
+            if (attrs.isRegularFile()) {
+                return FileType.REGULAR_FILE;
+            } else if (attrs.isDirectory()) {
+                return FileType.DIR;
+            } else if (attrs.isSymbolicLink()) {
+                return FileType.SYMLINK;
+            }
+            return FileType.OTHER;
+        }
+    }
+
+    public static int FILE_TYPE_FLAG = 0170000;
+}

os/src/ZipOps.scala

@@ -1,5 +1,11 @@
 package os
 
+import org.apache.commons.compress.archivers.zip.{
+  ZipArchiveEntry,
+  ZipFile => ZipArchiveFile,
+  ZipArchiveOutputStream
+}
+
 import java.net.URI
 import java.nio.file.{FileSystem, FileSystems, Files}
 import java.nio.file.attribute.{BasicFileAttributeView, FileTime, PosixFilePermissions}
@@ -97,8 +103,14 @@ object zip {
   ): Unit = {
     sources.foreach { source =>
       if (os.isDir(source.src)) {
-        for (path <- os.walk(source.src)) {
-          if (os.isFile(path) && shouldInclude(path.toString, excludePatterns, includePatterns)) {
+        val contents = os.walk(source.src)
+        if (contents.isEmpty)
+          makeZipEntry0(source.src, source.dest.getOrElse(os.sub / source.src.last))
+        for (path <- contents) {
+          if (
+            (os.isFile(path) && shouldInclude(path.toString, excludePatterns, includePatterns)) ||
+            (os.isDir(path) && os.walk.stream(path).headOption.isEmpty)
+          ) {
             makeZipEntry0(path, source.dest.getOrElse(os.sub) / path.subRelativeTo(source.src))
           }
         }
@@ -115,7 +127,7 @@ object zip {
       compressionLevel: Int,
       out: java.io.OutputStream
   ): Unit = {
-    val zipOut = new ZipOutputStream(out)
+    val zipOut = new ZipArchiveOutputStream(out)
     zipOut.setLevel(compressionLevel)
 
     try {
@@ -125,6 +137,7 @@ object zip {
         includePatterns,
         (path, sub) => makeZipEntry(path, sub, preserveMtimes, zipOut)
       )
+      zipOut.finish()
     } finally {
       zipOut.close()
     }
@@ -143,35 +156,41 @@ object zip {
     !isExcluded && isIncluded
   }
 
+  private def toFileType(file: os.Path): PermissionUtils.FileType = {
+    if (os.isLink(file)) PermissionUtils.FileType.SYMLINK
+    else if (os.isFile(file)) PermissionUtils.FileType.REGULAR_FILE
+    else if (os.isDir(file)) PermissionUtils.FileType.DIR
+    else PermissionUtils.FileType.OTHER
+  }
+
   private def makeZipEntry(
       file: os.Path,
       sub: os.SubPath,
       preserveMtimes: Boolean,
-      zipOut: ZipOutputStream
+      zipOut: ZipArchiveOutputStream
   ) = {
+    val name =
+      if (os.isDir(file)) sub.toString() + "/"
+      else sub.toString()
+    val zipEntry = new ZipArchiveEntry(name)
 
-    val mtimeOpt = if (preserveMtimes) Some(os.mtime(file)) else None
-
-    val fis = if (os.isFile(file)) Some(os.read.inputStream(file)) else None
-    try makeZipEntry0(sub, fis, mtimeOpt, zipOut)
-    finally fis.foreach(_.close())
-  }
+    val mtime = if (preserveMtimes) os.mtime(file) else 0
+    zipEntry.setTime(mtime)
 
-  private def makeZipEntry0(
-      sub: os.SubPath,
-      is: Option[java.io.InputStream],
-      preserveMtimes: Option[Long],
-      zipOut: ZipOutputStream
-  ) = {
-    val zipEntry = new ZipEntry(sub.toString)
+    val mode = PermissionUtils.modeFromPermissions(os.perms(file).toSet(), toFileType(file))
+    zipEntry.setUnixMode(mode)
 
-    preserveMtimes match {
-      case Some(mtime) => zipEntry.setTime(mtime)
-      case None => zipEntry.setTime(0)
-    }
+    val fis =
+      if (os.isLink(file))
+        Some(new java.io.ByteArrayInputStream(os.readLink(file).toString().getBytes()))
+      else if (os.isFile(file)) Some(os.read.inputStream(file))
+      else None
 
-    zipOut.putNextEntry(zipEntry)
-    is.foreach(os.Internals.transfer(_, zipOut, close = false))
+    try {
+      zipOut.putArchiveEntry(zipEntry)
+      fis.foreach(os.Internals.transfer(_, zipOut, close = false))
+      zipOut.closeArchiveEntry()
+    } finally fis.foreach(_.close())
   }
 
   /**
@@ -254,7 +273,39 @@ object unzip {
       excludePatterns: Seq[Regex] = List(),
       includePatterns: Seq[Regex] = List()
   ): os.Path = {
-    stream(os.read.stream(source), dest, excludePatterns, includePatterns)
+    checker.value.onWrite(dest)
+
+    val zipFile = new ZipArchiveFile(source.toIO)
+    val zipEntryInputStreams = zipFile.getEntries.asScala
+      .filter(ze => os.zip.shouldInclude(ze.getName, excludePatterns, includePatterns))
+      .map(ze => (ze, zipFile.getInputStream(ze)))
+
+    try {
+      for ((zipEntry, zipInputStream) <- zipEntryInputStreams) {
+        val newFile = dest / os.SubPath(zipEntry.getName)
+        val mode = zipEntry.getUnixMode
+        val perms = if (mode > 0 && !scala.util.Properties.isWin) {
+          os.PermSet.fromSet(PermissionUtils.permissionsFromMode(mode))
+        } else null
+
+        if (zipEntry.isDirectory) {
+          os.makeDir.all(newFile, perms = perms)
+        } else if (zipEntry.isUnixSymlink() && !scala.util.Properties.isWin) {
+          val target = scala.io.Source.fromInputStream(zipInputStream).mkString
+          val path = java.nio.file.Paths.get(target)
+          val dest = if (path.isAbsolute) os.Path(path) else os.RelPath(path)
+          os.symlink(newFile, dest)
+        } else {
+          val outputStream = os.write.outputStream(newFile, createFolders = true)
+          os.Internals.transfer(zipInputStream, outputStream, close = false)
+          outputStream.close()
+          if (perms != null) os.perms.set(newFile, perms)
+        }
+      }
+    } finally {
+      zipFile.close()
+    }
+
     dest
   }