diff --git a/build.mill b/build.mill index 4015fa3a..701175df 100644 --- a/build.mill +++ b/build.mill @@ -26,6 +26,7 @@ object Deps { val sourcecode = ivy"com.lihaoyi::sourcecode::0.4.2" val utest = ivy"com.lihaoyi::utest::0.8.4" val expecty = ivy"com.eed3si9n.expecty::expecty::0.16.0" + val apacheCommonsCompress = ivy"org.apache.commons:commons-compress:1.27.1" def scalaReflect(scalaVersion: String) = ivy"org.scala-lang:scala-reflect:$scalaVersion" def scalaLibrary(version: String) = ivy"org.scala-lang:scala-library:${version}" } @@ -112,7 +113,7 @@ trait OsLibModule } trait OsModule extends OsLibModule { outer => - def ivyDeps = Agg(Deps.geny) + def ivyDeps = Agg(Deps.geny, Deps.apacheCommonsCompress) override def compileIvyDeps = T { val scalaReflectOpt = Option.when(!ZincWorkerUtil.isDottyOrScala3(scalaVersion()))( Deps.scalaReflect(scalaVersion()) diff --git a/os/src/PermissionUtils.java b/os/src/PermissionUtils.java new file mode 100644 index 00000000..9c555d19 --- /dev/null +++ b/os/src/PermissionUtils.java @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// taken from https://github.com/apache/ant/blob/ecfca50b0133c576021dd088f855ee878a6b8e66/src/main/org/apache/tools/ant/util/PermissionUtils.java +package os; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.BasicFileAttributes; +import java.nio.file.attribute.PosixFileAttributeView; +import java.nio.file.attribute.PosixFilePermission; +import java.util.EnumSet; +import java.util.Set; + +/** + * Contains helper methods for dealing with {@link + * PosixFilePermission} or the traditional Unix mode representation of + * permissions. + */ +class PermissionUtils { + + private PermissionUtils() { } + + /** + * Translates a set of permissions into a Unix stat(2) {@code + * st_mode} result. + * @param permissions the permissions + * @param type the file type + * @return the "mode" + */ + public static int modeFromPermissions(Set permissions, + FileType type) { + int mode; + switch (type) { + case SYMLINK: + mode = 012; + break; + case REGULAR_FILE: + mode = 010; + break; + case DIR: + mode = 004; + break; + default: + // OTHER could be a character or block device, a socket or a FIFO - so don't set anything + mode = 0; + break; + } + mode <<= 3; + mode <<= 3; // we don't support sticky, setuid, setgid + mode |= modeFromPermissions(permissions, "OWNER"); + mode <<= 3; + mode |= modeFromPermissions(permissions, "GROUP"); + mode <<= 3; + mode |= modeFromPermissions(permissions, "OTHERS"); + return mode; + } + + /** + * Translates a Unix stat(2) {@code st_mode} compatible value into + * a set of permissions. + * @param mode the "mode" + * @return set of permissions + */ + public static Set permissionsFromMode(int mode) { + Set permissions = EnumSet.noneOf(PosixFilePermission.class); + addPermissions(permissions, "OTHERS", mode); + addPermissions(permissions, "GROUP", mode >> 3); + addPermissions(permissions, "OWNER", mode >> 6); + return permissions; + } + + private static long modeFromPermissions(Set permissions, + String prefix) { + long mode = 0; + if (permissions.contains(PosixFilePermission.valueOf(prefix + "_READ"))) { + mode |= 4; + } + if (permissions.contains(PosixFilePermission.valueOf(prefix + "_WRITE"))) { + mode |= 2; + } + if (permissions.contains(PosixFilePermission.valueOf(prefix + "_EXECUTE"))) { + mode |= 1; + } + return mode; + } + + private static void addPermissions(Set permissions, + String prefix, long mode) { + if ((mode & 1) == 1) { + permissions.add(PosixFilePermission.valueOf(prefix + "_EXECUTE")); + } + if ((mode & 2) == 2) { + permissions.add(PosixFilePermission.valueOf(prefix + "_WRITE")); + } + if ((mode & 4) == 4) { + permissions.add(PosixFilePermission.valueOf(prefix + "_READ")); + } + } + + /** + * The supported types of files, maps to the {@code isFoo} methods + * in {@link java.nio.file.attribute.BasicFileAttributes}. + */ + public enum FileType { + /** A regular file. */ + REGULAR_FILE, + /** A directory. */ + DIR, + /** A symbolic link. */ + SYMLINK, + /** Something that is neither a regular file nor a directory nor a symbolic link. */ + OTHER; + + /** + * Determines the file type of a {@link Path}. + * + * @param p Path + * @return FileType + * @throws IOException if file attributes cannot be read + */ + public static FileType of(Path p) throws IOException { + BasicFileAttributes attrs = + Files.readAttributes(p, BasicFileAttributes.class); + if (attrs.isRegularFile()) { + return FileType.REGULAR_FILE; + } else if (attrs.isDirectory()) { + return FileType.DIR; + } else if (attrs.isSymbolicLink()) { + return FileType.SYMLINK; + } + return FileType.OTHER; + } + } + + public static int FILE_TYPE_FLAG = 0170000; +} diff --git a/os/src/ZipOps.scala b/os/src/ZipOps.scala index 317b27a8..41d0fab5 100644 --- a/os/src/ZipOps.scala +++ b/os/src/ZipOps.scala @@ -1,5 +1,11 @@ package os +import org.apache.commons.compress.archivers.zip.{ + ZipArchiveEntry, + ZipFile => ZipArchiveFile, + ZipArchiveOutputStream +} + import java.net.URI import java.nio.file.{FileSystem, FileSystems, Files} import java.nio.file.attribute.{BasicFileAttributeView, FileTime, PosixFilePermissions} @@ -97,8 +103,14 @@ object zip { ): Unit = { sources.foreach { source => if (os.isDir(source.src)) { - for (path <- os.walk(source.src)) { - if (os.isFile(path) && shouldInclude(path.toString, excludePatterns, includePatterns)) { + val contents = os.walk(source.src) + if (contents.isEmpty) + makeZipEntry0(source.src, source.dest.getOrElse(os.sub / source.src.last)) + for (path <- contents) { + if ( + (os.isFile(path) && shouldInclude(path.toString, excludePatterns, includePatterns)) || + (os.isDir(path) && os.walk.stream(path).headOption.isEmpty) + ) { makeZipEntry0(path, source.dest.getOrElse(os.sub) / path.subRelativeTo(source.src)) } } @@ -115,7 +127,7 @@ object zip { compressionLevel: Int, out: java.io.OutputStream ): Unit = { - val zipOut = new ZipOutputStream(out) + val zipOut = new ZipArchiveOutputStream(out) zipOut.setLevel(compressionLevel) try { @@ -125,6 +137,7 @@ object zip { includePatterns, (path, sub) => makeZipEntry(path, sub, preserveMtimes, zipOut) ) + zipOut.finish() } finally { zipOut.close() } @@ -143,35 +156,41 @@ object zip { !isExcluded && isIncluded } + private def toFileType(file: os.Path): PermissionUtils.FileType = { + if (os.isLink(file)) PermissionUtils.FileType.SYMLINK + else if (os.isFile(file)) PermissionUtils.FileType.REGULAR_FILE + else if (os.isDir(file)) PermissionUtils.FileType.DIR + else PermissionUtils.FileType.OTHER + } + private def makeZipEntry( file: os.Path, sub: os.SubPath, preserveMtimes: Boolean, - zipOut: ZipOutputStream + zipOut: ZipArchiveOutputStream ) = { + val name = + if (os.isDir(file)) sub.toString() + "/" + else sub.toString() + val zipEntry = new ZipArchiveEntry(name) - val mtimeOpt = if (preserveMtimes) Some(os.mtime(file)) else None - - val fis = if (os.isFile(file)) Some(os.read.inputStream(file)) else None - try makeZipEntry0(sub, fis, mtimeOpt, zipOut) - finally fis.foreach(_.close()) - } + val mtime = if (preserveMtimes) os.mtime(file) else 0 + zipEntry.setTime(mtime) - private def makeZipEntry0( - sub: os.SubPath, - is: Option[java.io.InputStream], - preserveMtimes: Option[Long], - zipOut: ZipOutputStream - ) = { - val zipEntry = new ZipEntry(sub.toString) + val mode = PermissionUtils.modeFromPermissions(os.perms(file).toSet(), toFileType(file)) + zipEntry.setUnixMode(mode) - preserveMtimes match { - case Some(mtime) => zipEntry.setTime(mtime) - case None => zipEntry.setTime(0) - } + val fis = + if (os.isLink(file)) + Some(new java.io.ByteArrayInputStream(os.readLink(file).toString().getBytes())) + else if (os.isFile(file)) Some(os.read.inputStream(file)) + else None - zipOut.putNextEntry(zipEntry) - is.foreach(os.Internals.transfer(_, zipOut, close = false)) + try { + zipOut.putArchiveEntry(zipEntry) + fis.foreach(os.Internals.transfer(_, zipOut, close = false)) + zipOut.closeArchiveEntry() + } finally fis.foreach(_.close()) } /** @@ -254,7 +273,39 @@ object unzip { excludePatterns: Seq[Regex] = List(), includePatterns: Seq[Regex] = List() ): os.Path = { - stream(os.read.stream(source), dest, excludePatterns, includePatterns) + checker.value.onWrite(dest) + + val zipFile = new ZipArchiveFile(source.toIO) + val zipEntryInputStreams = zipFile.getEntries.asScala + .filter(ze => os.zip.shouldInclude(ze.getName, excludePatterns, includePatterns)) + .map(ze => (ze, zipFile.getInputStream(ze))) + + try { + for ((zipEntry, zipInputStream) <- zipEntryInputStreams) { + val newFile = dest / os.SubPath(zipEntry.getName) + val mode = zipEntry.getUnixMode + val perms = if (mode > 0 && !scala.util.Properties.isWin) { + os.PermSet.fromSet(PermissionUtils.permissionsFromMode(mode)) + } else null + + if (zipEntry.isDirectory) { + os.makeDir.all(newFile, perms = perms) + } else if (zipEntry.isUnixSymlink() && !scala.util.Properties.isWin) { + val target = scala.io.Source.fromInputStream(zipInputStream).mkString + val path = java.nio.file.Paths.get(target) + val dest = if (path.isAbsolute) os.Path(path) else os.RelPath(path) + os.symlink(newFile, dest) + } else { + val outputStream = os.write.outputStream(newFile, createFolders = true) + os.Internals.transfer(zipInputStream, outputStream, close = false) + outputStream.close() + if (perms != null) os.perms.set(newFile, perms) + } + } + } finally { + zipFile.close() + } + dest }