diff --git a/Readme.adoc b/Readme.adoc index faf9bfbb..ecc02401 100644 --- a/Readme.adoc +++ b/Readme.adoc @@ -1233,7 +1233,8 @@ def apply(dest: os.Path, includePatterns: Seq[Regex] = List(), preserveMtimes: Boolean = false, deletePatterns: Seq[Regex] = List(), - compressionLevel: Int = -1 /* 0-9 */): os.Path + compressionLevel: Int = -1, /* 0-9 */ + followLinks: Boolean = true): os.Path ---- The zip object provides functionality to create or modify zip archives. It supports: @@ -1243,14 +1244,19 @@ The zip object provides functionality to create or modify zip archives. It suppo - Exclude Patterns (-x): You can specify files or patterns to exclude while zipping. - Include Patterns (-i): You can include specific files or patterns while zipping. - Delete Patterns (-d): You can delete specific files from an existing zip archive. -- Configuring whether or not to preserve filesyste mtimes and permissions +- Symbolic Links (-y): You can configure to zip symbolic links as symbolic links on Linux/Unix by setting `followLinks = false`. Symbolic links are zipped as the referenced files by default on Linux/Unix, and always on Windows. +- Configuring whether or not to preserve filesyste mtimes. +- Preserving Unix file permissions. This will create a new zip archive at `dest` containing `file1.txt` and everything inside `sources`. If `dest` already exists as a zip, the files will be appended to the existing zip, and any existing zip entries matching `deletePatterns` will be removed. -Note that `os.zip` doesn't support creating/unpacking symlinks or filesystem permissions -in Zip files, because the underlying `java.util.zip.Zip*Stream` doesn't support them. +When modifying an existing zip file, +- Unix file permissions will be preserved if Java Runtime Version >= 14. +- If using Java Runtime Version < 14, Unix file permissions are not preserved, even for existing zip entries. +- Symbolics links will always be stored as the referenced files. +- Existing symbolic links stored in the zip might lose their symbolic link file type field and become broken. ===== Zipping Files and Folders @@ -1375,6 +1381,8 @@ assert(paths == Seq(unzippedFolder / "File.txt")) This can be useful for streaming the zipped data to places which are not files: over the network, over a pipe, etc. +File permissions will be preserved. Symbolic links will be zipped as the referenced files by default on Linux/Unix, and always on Windows. To zip them as symbolic links on Linux/Unix, set `followLinks = false`. + ==== `os.unzip` ===== Unzipping Files @@ -1384,7 +1392,7 @@ over the network, over a pipe, etc. os.unzip(os.Path("/path/to/archive.zip"), Some(os.Path("/path/to/destination"))) ---- -This extracts the contents of `archive.zip` to the specified destination. +This extracts the contents of `archive.zip` to the specified destination. It supports preserving file permissions and symbolic links. ===== Excluding Files While Unzipping @@ -1407,7 +1415,7 @@ You can list the contents of the zip file without extracting them: os.unzip.list(os.Path("/path/to/archive.zip")) ---- -This will print all the file paths contained in the zip archive. +This will print all the file paths contained in the zip archive. File permissions and symbolic links will not be preserved. ==== `os.unzip.stream` @@ -1427,6 +1435,9 @@ os.unzip.stream( This can be useful if the zip file does not exist on disk, e.g. if it is received over the network or produced in-memory by application logic. +File permissions and symbolic links are not supported since permissions and symlink mode are stored as external attributes which might reside in the central directory located at the end of the zip archive. +For more a more detailed explanation see the `ZipArchiveInputStream` vs `ZipFile` section at https://commons.apache.org/proper/commons-compress/zip.html. + OS-Lib also provides the `os.unzip.streamRaw` API, which is a lower level API used internally within `os.unzip.stream` but can also be used directly if lower-level control is necessary. @@ -1464,6 +1475,8 @@ finally zipFile3.close() of the zip file rather than a bare path on the filesystem. Note that you need to call `ZipRoot#close()` when you are done with it to avoid leaking filesystem resources. +File permissions are only supported for Java Runtime Version >= 14. Symbolic links are not supported. Using `os.zip.open` on a zip archive that contains symbolic links might break the links. + === Filesystem Metadata ==== `os.stat` @@ -1793,7 +1806,7 @@ is run: * `cwd`: the working directory of the subprocess * `env`: any additional environment variables you wish to set in the subprocess - in addition to those passed via `propagateEnv`. You can also set their values + in addition to those passed via `propagateEnv`. You can also set their values to `null` to remove specific variables. * `stdin`: any data you wish to pass to the subprocess's standard input * `stdout`/`stderr`: these are ``os.Redirect``s that let you configure how the diff --git a/build.mill b/build.mill index bf85a854..0ec4e0c2 100644 --- a/build.mill +++ b/build.mill @@ -23,6 +23,7 @@ object Deps { val acyclic = ivy"com.lihaoyi:::acyclic:0.3.18" val jna = ivy"net.java.dev.jna:jna:5.15.0" val geny = ivy"com.lihaoyi::geny::1.1.1" + val ant = ivy"org.apache.ant:ant:1.10.15" val sourcecode = ivy"com.lihaoyi::sourcecode::0.4.2" val utest = ivy"com.lihaoyi::utest::0.8.4" val expecty = ivy"com.eed3si9n.expecty::expecty::0.16.0" @@ -78,13 +79,7 @@ trait MiMaChecks extends Mima { ) } -trait OsLibModule - extends CrossScalaModule - with PublishModule - with AcyclicModule - with SafeDeps - with PlatformScalaModule { outer => - +trait OsLibPublishModule extends PublishModule { def publishVersion = VcsVersion.vcsState().format() def pomSettings = PomSettings( description = artifactName(), @@ -99,6 +94,14 @@ trait OsLibModule Developer("lihaoyi", "Li Haoyi", "https://github.com/lihaoyi") ) ) +} + +trait OsLibModule + extends OsLibPublishModule + with CrossScalaModule + with AcyclicModule + with SafeDeps + with PlatformScalaModule { outer => trait OsLibTestModule extends ScalaModule with TestModule.Utest with SafeDeps { def ivyDeps = Agg(Deps.utest, Deps.sourcecode) @@ -169,6 +172,8 @@ object os extends Module { object jvm extends Cross[OsJvmModule](scalaVersions) trait OsJvmModule extends OsModule with MiMaChecks { + def moduleDeps = super.moduleDeps ++ Seq(os.zip) + object test extends ScalaTests with OsLibTestModule { override def ivyDeps = T { super.ivyDeps() ++ Agg(Deps.expecty) } @@ -194,6 +199,55 @@ object os extends Module { object nohometest extends ScalaTests with OsLibTestModule } + object zip extends JavaModule with OsLibPublishModule { + def apacheAntZipOriginalSource: T[PathRef] = Task(persistent = true) { + if (!_root_.os.exists(Task.dest / "unzipped")) { + val antVersion = Deps.ant.version + _root_.os.unzip.stream( + requests.get.stream( + s"https://repo1.maven.org/maven2/org/apache/ant/ant/$antVersion/ant-$antVersion-sources.jar" + ), + Task.dest / "unzipped" + ) + } + + PathRef(Task.dest / "unzipped" / "org/apache/tools/zip") + } + + /** + * Shades Apache Ant + * [[`org.apache.tools.zip` https://ant.apache.org/manual/api/org/apache/tools/zip/package-summary.html package]] to + * provide Unix file permission and symbolic link support for `os.zip` and `os.unzip` + * + * A third party dependency is needed since JDK's own + * [[`jdk.zipfs` https://docs.oracle.com/en/java/javase/14/docs/api/jdk.zipfs/module-summary.html]] does not support + * symbolic links and only supports file permissions since JDK 14. + * + * Apache Ant `org.apache.tools.zip` was chosen over Apache Commons Compress due to the former not having any + * third party dependency, only depending on Java core libraries while the later also depends on Apache Commons IO. + * + * To avoid classpath conflicts, the dependency is shaded and compiled from source. Only the `org.apache.tools.zip` + * package, not the entire Ant codebase, is needed. This only adds < 100kb to Os-Lib jar size. + */ + def generatedSources = T { + val pkg = "os.shaded_org_apache_tools_zip" + val zipSrc = T.dest / "os/shaded_org_apache_tools_zip" + _root_.os.makeDir.all(zipSrc) + + // Move from "package org.apache.tools.zip" to "package os.shaded_org_apache_tools_zip" + // Make all classes package private (private [os]) by removing any `public` access modifier + _root_.os.walk.stream(apacheAntZipOriginalSource().path) + .filter(_.ext == "java") + .foreach { p => + val content = _root_.os.read(p) + .replaceAll("org.apache.tools.zip", pkg) + _root_.os.write(zipSrc / p.last, content) + } + + Seq(PathRef(T.dest)) + } + } + /*object native extends Cross[OsNativeModule](scalaVersions) trait OsNativeModule extends OsModule with ScalaNativeModule { def scalaNativeVersion = "0.5.2" diff --git a/os/src/ZipOps.scala b/os/src/ZipOps.scala index bfa8d077..e8ee40a0 100644 --- a/os/src/ZipOps.scala +++ b/os/src/ZipOps.scala @@ -1,11 +1,20 @@ package os +import os.{shaded_org_apache_tools_zip => apache} + import java.net.URI -import java.nio.file.{FileSystem, FileSystems, Files} -import java.nio.file.attribute.{BasicFileAttributeView, FileTime, PosixFilePermissions} +import java.nio.file.{FileSystem, FileSystemException, FileSystems, Files, LinkOption} +import java.nio.file.attribute.{ + BasicFileAttributes, + BasicFileAttributeView, + FileTime, + PosixFilePermission, + PosixFilePermissions +} import java.util.zip.{ZipEntry, ZipFile, ZipInputStream, ZipOutputStream} import scala.collection.JavaConverters._ import scala.util.matching.Regex +import scala.util.Properties.isWin object zip { @@ -17,15 +26,21 @@ object zip { def open(path: Path): ZipRoot = { new ZipRoot(FileSystems.newFileSystem( new URI("jar", path.wrapped.toUri.toString, null), - Map("create" -> "true").asJava + Map("create" -> "true", "enablePosixFileAttributes" -> "true").asJava )) } /** * Zips the provided list of files and directories into a single ZIP archive. * + * Unix file permissions will be preserved when creating a new zip, i.e. when `dest` does not already exists. + * * If `dest` already exists and is a zip, performs modifications to `dest` in place - * rather than creating a new zip. + * rather than creating a new zip. In that case, + * - Unix file permissions will be preserved if Java Runtime Version >= 14 + * - if using Java Runtime Version < 14, Unix file permissions are not preserved, even for existing zip entries + * - symbolics links will always be stored as the referenced files + * - existing symbolic links stored in the zip might lose their symbolic link file type field and become broken * * @param dest The path to the destination ZIP file. * @param sources A list of paths to files and directories to be zipped. Defaults to an empty list. @@ -33,7 +48,8 @@ object zip { * @param includePatterns A list of regular expression patterns to include files in the ZIP archive. Defaults to an empty list (includes all files). * @param preserveMtimes Whether to preserve modification times (mtimes) of the files. * @param deletePatterns A list of regular expression patterns to delete files from an existing ZIP archive before appending new ones. - * @param compressionLevel number from 0-9, where 0 is no compression and 9 is best compression. Defaults to -1 (default compression) + * @param compressionLevel number from 0-9, where 0 is no compression and 9 is best compression. Defaults to -1 (default compression). + * @param followLinks Whether to store symbolic links as the referenced files. Default to `true`. Setting this to `false` has no effect when modifying a zip file in place. * @return The path to the created ZIP archive. */ def apply( @@ -43,7 +59,8 @@ object zip { includePatterns: Seq[Regex] = List(), preserveMtimes: Boolean = false, deletePatterns: Seq[Regex] = List(), - compressionLevel: Int = java.util.zip.Deflater.DEFAULT_COMPRESSION + compressionLevel: Int = java.util.zip.Deflater.DEFAULT_COMPRESSION, + followLinks: Boolean = true ): os.Path = { checker.value.onWrite(dest) // check read preemptively in case "dest" is created @@ -62,26 +79,36 @@ object zip { excludePatterns, includePatterns, (path, sub) => { - os.copy(path, opened / sub, createFolders = true) + val dest = opened / sub + + if (os.isDir(path)) + os.makeDir.all(dest) + else + os.copy(path, dest, createFolders = true) + + if (!isWin && Runtime.version.feature >= 14) + Files.setPosixFilePermissions(dest.wrapped, os.perms(path).toSet()) + if (!preserveMtimes) { - os.mtime.set(opened / sub, 0) + os.mtime.set(dest, 0) // This is the only way we can properly zero out filesystem metadata within the // Zip file filesystem; `os.mtime.set` is not enough val view = - Files.getFileAttributeView((opened / sub).toNIO, classOf[BasicFileAttributeView]) + Files.getFileAttributeView(dest.wrapped, classOf[BasicFileAttributeView]) view.setTimes(FileTime.fromMillis(0), FileTime.fromMillis(0), FileTime.fromMillis(0)) } } ) } finally opened.close() } else { - val f = Files.newOutputStream(dest.toNIO) + val f = Files.newOutputStream(dest.wrapped) try createNewZip( sources, excludePatterns, includePatterns, preserveMtimes, compressionLevel, + followLinks, f ) finally f.close() @@ -98,16 +125,13 @@ object zip { sources.foreach { source => if (os.isDir(source.src)) { val contents = os.walk(source.src) - if (contents.isEmpty) - source.dest - .filter(_ => shouldInclude(source.src.toString + "/", excludePatterns, includePatterns)) - .foreach(makeZipEntry0(source.src, _)) + source.dest + .filter(_ => shouldInclude(source.src.toString + "/", excludePatterns, includePatterns)) + .foreach(makeZipEntry0(source.src, _)) for (path <- contents) { if ( (os.isFile(path) && shouldInclude(path.toString, excludePatterns, includePatterns)) || - (os.isDir(path) && - os.walk.stream(path).headOption.isEmpty && - shouldInclude(path.toString + "/", excludePatterns, includePatterns)) + (os.isDir(path) && shouldInclude(path.toString + "/", excludePatterns, includePatterns)) ) { makeZipEntry0(path, source.dest.getOrElse(os.sub) / path.subRelativeTo(source.src)) } @@ -123,9 +147,10 @@ object zip { includePatterns: Seq[Regex], preserveMtimes: Boolean, compressionLevel: Int, + resolveLinks: Boolean, out: java.io.OutputStream ): Unit = { - val zipOut = new ZipOutputStream(out) + val zipOut = new apache.ZipOutputStream(out) zipOut.setLevel(compressionLevel) try { @@ -133,7 +158,7 @@ object zip { sources, excludePatterns, includePatterns, - (path, sub) => makeZipEntry(path, sub, preserveMtimes, zipOut) + (path, sub) => makeZipEntry(path, sub, preserveMtimes, resolveLinks, zipOut) ) zipOut.finish() } finally { @@ -154,21 +179,61 @@ object zip { !isExcluded && isIncluded } + private def toFileType( + file: os.Path, + followLinks: Boolean = false + ): apache.PermissionUtils.FileType = { + val attrs = if (followLinks) + Files.readAttributes(file.wrapped, classOf[BasicFileAttributes]) + else Files.readAttributes(file.wrapped, classOf[BasicFileAttributes], LinkOption.NOFOLLOW_LINKS) + + if (attrs.isSymbolicLink()) apache.PermissionUtils.FileType.SYMLINK + else if (attrs.isRegularFile()) apache.PermissionUtils.FileType.REGULAR_FILE + else if (attrs.isDirectory()) apache.PermissionUtils.FileType.DIR + else apache.PermissionUtils.FileType.OTHER + } + + // In zip, symlink info and posix permissions are stored together thus to store symlinks as + // symlinks on Windows some permissions need to be set as well. Use 644/"rw-r--r--" as the default. + private lazy val defaultPermissions = Set( + PosixFilePermission.OWNER_READ, + PosixFilePermission.OWNER_WRITE, + PosixFilePermission.GROUP_READ, + PosixFilePermission.OTHERS_READ + ).asJava + private def makeZipEntry( file: os.Path, sub: os.SubPath, preserveMtimes: Boolean, - zipOut: ZipOutputStream + followLinks: Boolean, + zipOut: apache.ZipOutputStream ) = { val name = if (os.isDir(file)) sub.toString + "/" else sub.toString - val zipEntry = new ZipEntry(name) + val zipEntry = new apache.ZipEntry(name) val mtime = if (preserveMtimes) os.mtime(file) else 0 zipEntry.setTime(mtime) - val fis = if (os.isFile(file)) Some(os.read.inputStream(file)) else None + val symlink = !followLinks && os.isLink(file) + + if (!isWin || symlink) { + val perms = + if (isWin) defaultPermissions else os.perms(file, followLinks = followLinks).toSet() + val mode = apache.PermissionUtils.modeFromPermissions( + perms, + toFileType(file, followLinks = followLinks) + ) + zipEntry.setUnixMode(mode) + } + + val fis = + if (symlink) + Some(new java.io.ByteArrayInputStream(os.readLink(file).toString().getBytes())) + else if (os.isFile(file)) Some(os.read.inputStream(file)) + else None try { zipOut.putNextEntry(zipEntry) @@ -186,6 +251,8 @@ object zip { * @param excludePatterns A list of regular expression patterns to exclude files during zipping. Defaults to an empty list. * @param includePatterns A list of regular expression patterns to include files in the ZIP archive. Defaults to an empty list (includes all files). * @param preserveMtimes Whether to preserve modification times (mtimes) of the files. + * @param compressionLevel number from 0-9, where 0 is no compression and 9 is best compression. Defaults to -1 (default compression). + * @param followLinks Whether to store symbolic links as the referenced files. Default to true. * @return A geny.Writable object for writing the ZIP data. */ def stream( @@ -193,7 +260,8 @@ object zip { excludePatterns: Seq[Regex] = List(), includePatterns: Seq[Regex] = List(), preserveMtimes: Boolean = false, - compressionLevel: Int = java.util.zip.Deflater.DEFAULT_COMPRESSION + compressionLevel: Int = java.util.zip.Deflater.DEFAULT_COMPRESSION, + followLinks: Boolean = false ): geny.Writable = { (outputStream: java.io.OutputStream) => { @@ -203,6 +271,7 @@ object zip { includePatterns, preserveMtimes, compressionLevel, + followLinks, outputStream ) } @@ -244,6 +313,9 @@ object unzip { } yield os.SubPath(zipEntry.getName) } + private def isSymLink(mode: Int): Boolean = + (mode & apache.PermissionUtils.FILE_TYPE_FLAG) == apache.UnixStat.LINK_FLAG + /** * Extract the given zip file into the destination directory * @@ -257,13 +329,83 @@ object unzip { excludePatterns: Seq[Regex] = List(), includePatterns: Seq[Regex] = List() ): os.Path = { - stream(os.read.stream(source), dest, excludePatterns, includePatterns) + checker.value.onWrite(dest) + + val zipFile = new apache.ZipFile(source.toIO) + val zipEntryInputStreams = zipFile.getEntries.asScala + .filter(ze => os.zip.shouldInclude(ze.getName, excludePatterns, includePatterns)) + .map(ze => { + val mode = ze.getUnixMode + ( + ze, + os.SubPath(ze.getName), + mode, + isSymLink(mode), + zipFile.getInputStream(ze) + ) + }) + .toList + .sortBy { case (_, path, _, isSymLink, _) => + // Unzipping symbolic links last. + // Enclosing directories are unzipped before their contents. + // This makes sure directory permissions are applied correctly. + (isSymLink, path) + } + + try { + for ((zipEntry, path, mode, isSymLink, zipInputStream) <- zipEntryInputStreams) { + val newFile = dest / path + val perms = if (mode > 0 && !isWin) { + os.PermSet.fromSet(apache.PermissionUtils.permissionsFromMode(mode)) + } else null + + if (zipEntry.isDirectory) { + os.makeDir.all(newFile, perms = perms) + if (perms != null && os.perms(newFile) != perms) { + // because of umask + os.perms.set(newFile, perms) + } + } else if (isSymLink) { + val target = scala.io.Source.fromInputStream(zipInputStream).mkString + val path = java.nio.file.Paths.get(target) + val dest = if (path.isAbsolute) os.Path(path) else os.RelPath(path) + os.makeDir.all(newFile / os.up) + try { + os.symlink(newFile, dest) + } catch { + case _: FileSystemException => { + System.err.println( + s"Failed to create symbolic link ${zipEntry.getName} -> ${target}.\n" + + (if (isWin) + "On Windows this might be due to lack of sufficient privilege or file system support.\n" + else "") + + "This zip entry will instead be unzipped as a file containing the target path." + ) + os.write(newFile, target) + } + } + } else { + val outputStream = os.write.outputStream(newFile, createFolders = true) + os.Internals.transfer(zipInputStream, outputStream, close = false) + outputStream.close() + if (!isWin && perms != null) os.perms.set(newFile, perms) + } + } + } finally { + zipFile.close() + } + dest } /** * Unzips a ZIP data stream represented by a geny.Readable and extracts it to a destination directory. * + * File permissions and symbolic links are not supported since permissions and symlink mode are stored + * as external attributes which reside in the central directory located at the end of the zip archive. + * For more a more detailed explanation see the `ZipArchiveInputStream` vs `ZipFile` section at + * [[https://commons.apache.org/proper/commons-compress/zip.html]]. + * * @param source A geny.Readable object representing the ZIP data stream. * @param dest The path to the destination directory for extracted files. * @param excludePatterns A list of regular expression patterns to exclude files during extraction. (Optional) diff --git a/os/test/src/ZipOpTests.scala b/os/test/src/ZipOpTests.scala index f7b9b99f..3462e550 100644 --- a/os/test/src/ZipOpTests.scala +++ b/os/test/src/ZipOpTests.scala @@ -256,6 +256,241 @@ object ZipOpTests extends TestSuite { assert(os.list(sources).toSet == expected) } + test("symLinkAndPermissions") { + def prepare( + wd: os.Path, + zipStream: Boolean = false, + unzipStream: Boolean = false, + followLinks: Boolean = true + ) = { + val zipFileName = "zipped.zip" + val source = wd / "folder2" + val link = os.rel / "nestedA" / "link.txt" + if (!scala.util.Properties.isWin) { + os.perms.set(source / "nestedA", os.PermSet.fromString("rwxrwxrwx")) + os.perms.set(source / "nestedA" / "a.txt", os.PermSet.fromString("rw-rw-rw-")) + os.symlink(source / link, os.rel / "a.txt") + } + + val zipped = + if (zipStream) { + os.write( + wd / zipFileName, + os.zip.stream(sources = List(source), followLinks = followLinks) + ) + wd / zipFileName + } else { + os.zip( + dest = wd / zipFileName, + sources = List(source), + followLinks = followLinks + ) + } + + val unzipped = + if (unzipStream) { + os.unzip.stream( + source = os.read.inputStream(zipped), + dest = wd / "unzipped" + ) + wd / "unzipped" + } else { + os.unzip( + dest = wd / "unzipped", + source = zipped + ) + } + + (source, unzipped, link) + } + + def walkRel(p: os.Path) = os.walk(p).map(_.relativeTo(p)) + + test("zip") - prep { wd => + if (!scala.util.Properties.isWin) { + val (source, unzipped, link) = prepare(wd, followLinks = true) + + // test all files are there + assert(walkRel(source).toSet == walkRel(unzipped).toSet) + // test all permissions are preserved + assert(os.walk.stream(source) + .filter(!os.isLink(_)) + .forall(p => os.perms(p) == os.perms(unzipped / p.relativeTo(source)))) + + // test symlinks are zipped as the referenced files + val unzippedLink = unzipped / link + assert(os.isFile(unzippedLink)) + assert(os.read(os.readLink.absolute(source / link)) == os.read(unzippedLink)) + } + } + + test("zipPreserveLinks") - prep { wd => + if (!scala.util.Properties.isWin) { + val (source, unzipped, link) = prepare(wd, followLinks = false) + + // test all files are there + assert(walkRel(source).toSet == walkRel(unzipped).toSet) + // test all permissions are preserved + assert(os.walk.stream(source) + .filter(!os.isLink(_)) + .forall(p => os.perms(p) == os.perms(unzipped / p.relativeTo(source)))) + + // test symlinks are zipped as symlinks + val unzippedLink = unzipped / link + assert(os.isLink(unzippedLink)) + assert(os.readLink(source / link) == os.readLink(unzippedLink)) + } + } + + test("zipStream") - prep { wd => + if (!scala.util.Properties.isWin) { + val (source, unzipped, link) = prepare(wd, zipStream = true, followLinks = true) + + // test all files are there + assert(walkRel(source).toSet == walkRel(unzipped).toSet) + // test all permissions are preserved + assert(os.walk.stream(source) + .filter(!os.isLink(_)) + .forall(p => os.perms(p) == os.perms(unzipped / p.relativeTo(source)))) + + // test symlinks are zipped as the referenced files + val unzippedLink = unzipped / link + assert(os.isFile(unzippedLink)) + assert(os.read(os.readLink.absolute(source / link)) == os.read(unzippedLink)) + } + } + + test("zipStreamPreserveLinks") - prep { wd => + if (!scala.util.Properties.isWin) { + val (source, unzipped, link) = prepare(wd, zipStream = true, followLinks = false) + + // test all files are there + assert(walkRel(source).toSet == walkRel(unzipped).toSet) + // test all permissions are preserved + assert(os.walk.stream(source) + .filter(!os.isLink(_)) + .forall(p => os.perms(p) == os.perms(unzipped / p.relativeTo(source)))) + + // test symlinks are zipped as symlinks + val unzippedLink = unzipped / link + assert(os.isLink(unzippedLink)) + assert(os.readLink(source / link) == os.readLink(unzippedLink)) + } + } + + test("unzipStreamWithLinks") - prep { wd => + if (!scala.util.Properties.isWin) { + val (source, unzipped, link) = prepare(wd, unzipStream = true, followLinks = false) + + // test all files are there + assert(walkRel(source).toSet == walkRel(unzipped).toSet) + + val unzippedLink = unzipped / link + assert(os.isFile(unzippedLink)) + assert(os.readLink(source / link).toString == os.read(unzippedLink)) + } + } + + test("unzipStream") - prep { wd => + if (!scala.util.Properties.isWin) { + val (source, unzipped, link) = prepare(wd, unzipStream = true, followLinks = true) + + // test all files are there + assert(walkRel(source).toSet == walkRel(unzipped).toSet) + + // test symlinks zipped as the referenced files are unzipped correctly + val unzippedLink = unzipped / link + assert(os.isFile(unzippedLink)) + assert(os.read(os.readLink.absolute(source / link)) == os.read(unzippedLink)) + } + } + + test("existingZip") - prep { wd => + if (!scala.util.Properties.isWin) { + val (source, unzipped, link) = prepare(wd) + + val newSource = os.pwd / "source" + os.makeDir(newSource) + + val newDir = newSource / "new" / "dir" + os.makeDir.all(newDir) + os.perms.set(newDir, os.PermSet.fromString("rwxrwxrwx")) + os.write.over(newDir / "a.txt", "Contents of a.txt") + + val newFile = os.sub / "new.txt" + val perms = os.PermSet.fromString("rw-rw-rw-") + os.write(newSource / newFile, "Contents of new.txt") + os.perms.set(newSource / newFile, perms) + + val newLink = os.sub / "newLink.txt" + os.symlink(newSource / newLink, os.rel / "new.txt") + + val newZipped = os.zip( + dest = wd / "zipped.zip", + sources = List(newSource) + ) + + val newUnzipped = os.unzip( + source = newZipped, + dest = wd / "newUnzipped" + ) + + // test all files are there + assert((walkRel(source) ++ walkRel(newSource)).toSet == walkRel(newUnzipped).toSet) + + // test permissions of existing zip entries are preserved + if (Runtime.version.feature >= 14) { + assert(os.walk.stream(source) + .filter(!os.isLink(_)) + .forall(p => os.perms(p) == os.perms(newUnzipped / p.relativeTo(source)))) + } + + // test existing symlinks zipped as the referenced files are unzipped + val unzippedNewLink = newUnzipped / newLink + assert(os.isFile(unzippedNewLink)) + assert(os.read(os.readLink.absolute(newSource / newLink)) == os.read(unzippedNewLink)) + + // test permissions of newly added files are preserved + val unzippedNewFile = newUnzipped / newFile + if (Runtime.version.feature >= 14) { + assert(os.perms(unzippedNewFile) == perms) + assert(os.perms(unzippedNewLink) == perms) + } + } + } + } + + test("zipSymlink") - prep { wd => + val zipFileName = "zipped.zip" + val source = wd / "folder1" + val linkName = "link.txt" + val link = os.rel / linkName + + os.symlink(source / link, os.rel / "one.txt") + + val zipped = os.zip( + dest = wd / zipFileName, + sources = List(source), + followLinks = false + ) + + val unzipped = os.unzip( + source = zipped, + dest = wd / "unzipped" + ) + + import os.{shaded_org_apache_tools_zip => apache} + val zipFile = new apache.ZipFile(zipped.toIO) + val entry = zipFile.getEntry(linkName) + + // check if zipped correctly as symlink + assert( + (entry.getUnixMode & apache.PermissionUtils.FILE_TYPE_FLAG) == apache.UnixStat.LINK_FLAG + ) + assert(os.isLink(unzipped / link)) + assert(os.readLink(unzipped / link) == os.readLink(source / link)) + } + test("unzipStream") - prep { wd => // Step 1: Create an in-memory ZIP file as a stream val zipStreamOutput = new ByteArrayOutputStream() diff --git a/os/zip/src/os/shaded_org_apache_tools_zip/PermissionUtils.java b/os/zip/src/os/shaded_org_apache_tools_zip/PermissionUtils.java new file mode 100644 index 00000000..60f08965 --- /dev/null +++ b/os/zip/src/os/shaded_org_apache_tools_zip/PermissionUtils.java @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// taken from https://github.com/apache/ant/blob/ecfca50b0133c576021dd088f855ee878a6b8e66/src/main/org/apache/tools/ant/util/PermissionUtils.java +// removed imports not from Java core libraries and code that uses them +package os.shaded_org_apache_tools_zip; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.BasicFileAttributes; +import java.nio.file.attribute.PosixFilePermission; +import java.util.EnumSet; +import java.util.Set; + +/** + * Contains helper methods for dealing with {@link + * PosixFilePermission} or the traditional Unix mode representation of + * permissions. + */ +public class PermissionUtils { + + private PermissionUtils() { } + + /** + * Translates a set of permissions into a Unix stat(2) {@code + * st_mode} result. + * @param permissions the permissions + * @param type the file type + * @return the "mode" + */ + public static int modeFromPermissions(Set permissions, + FileType type) { + int mode; + switch (type) { + case SYMLINK: + mode = 012; + break; + case REGULAR_FILE: + mode = 010; + break; + case DIR: + mode = 004; + break; + default: + // OTHER could be a character or block device, a socket or a FIFO - so don't set anything + mode = 0; + break; + } + mode <<= 3; + mode <<= 3; // we don't support sticky, setuid, setgid + mode |= modeFromPermissions(permissions, "OWNER"); + mode <<= 3; + mode |= modeFromPermissions(permissions, "GROUP"); + mode <<= 3; + mode |= modeFromPermissions(permissions, "OTHERS"); + return mode; + } + + /** + * Translates a Unix stat(2) {@code st_mode} compatible value into + * a set of permissions. + * @param mode the "mode" + * @return set of permissions + */ + public static Set permissionsFromMode(int mode) { + Set permissions = EnumSet.noneOf(PosixFilePermission.class); + addPermissions(permissions, "OTHERS", mode); + addPermissions(permissions, "GROUP", mode >> 3); + addPermissions(permissions, "OWNER", mode >> 6); + return permissions; + } + + private static long modeFromPermissions(Set permissions, + String prefix) { + long mode = 0; + if (permissions.contains(PosixFilePermission.valueOf(prefix + "_READ"))) { + mode |= 4; + } + if (permissions.contains(PosixFilePermission.valueOf(prefix + "_WRITE"))) { + mode |= 2; + } + if (permissions.contains(PosixFilePermission.valueOf(prefix + "_EXECUTE"))) { + mode |= 1; + } + return mode; + } + + private static void addPermissions(Set permissions, + String prefix, long mode) { + if ((mode & 1) == 1) { + permissions.add(PosixFilePermission.valueOf(prefix + "_EXECUTE")); + } + if ((mode & 2) == 2) { + permissions.add(PosixFilePermission.valueOf(prefix + "_WRITE")); + } + if ((mode & 4) == 4) { + permissions.add(PosixFilePermission.valueOf(prefix + "_READ")); + } + } + + /** + * The supported types of files, maps to the {@code isFoo} methods + * in {@link java.nio.file.attribute.BasicFileAttributes}. + */ + public enum FileType { + /** A regular file. */ + REGULAR_FILE, + /** A directory. */ + DIR, + /** A symbolic link. */ + SYMLINK, + /** Something that is neither a regular file nor a directory nor a symbolic link. */ + OTHER; + + /** + * Determines the file type of a {@link Path}. + * + * @param p Path + * @return FileType + * @throws IOException if file attributes cannot be read + */ + public static FileType of(Path p) throws IOException { + BasicFileAttributes attrs = + Files.readAttributes(p, BasicFileAttributes.class); + if (attrs.isRegularFile()) { + return FileType.REGULAR_FILE; + } else if (attrs.isDirectory()) { + return FileType.DIR; + } else if (attrs.isSymbolicLink()) { + return FileType.SYMLINK; + } + return FileType.OTHER; + } + } + + public static int FILE_TYPE_FLAG = 0170000; +}