diff --git a/Readme.adoc b/Readme.adoc index 103afbea..fe828e98 100644 --- a/Readme.adoc +++ b/Readme.adoc @@ -985,9 +985,9 @@ os.remove(target: Path, checkExists: Boolean = false): Boolean ---- Remove the target file or folder. Folders need to be empty to be removed; if you -want to remove a folder tree recursively, use <>. +want to remove a folder tree recursively, use <>. Returns `true` if the file was present before. -It will fail with an exception when the file is missing but `checkExists` is `true`, +It will fail with an exception when the file is missing but `checkExists` is `true`, or when the directory to remove is not empty. [source,scala] @@ -1215,6 +1215,133 @@ os.write(tempDir / "file", "Hello") os.list(tempDir) ==> Seq(tempDir / "file") ---- +=== Zip & Unzip Files + +==== `os.zip` + +[source,scala] +---- +os.zip(destination: os.Path, + listOfPaths: List[os.Path] = List(), + appendToExisting: Boolean = false, + excludePatterns: List[String] = List(), + includePatterns: List[String] = List(), + deletePatterns: List[String] = List(), + preserveMtimes: Boolean = true, + preservePermissions: Boolean = true): Path +---- + +The zip object provides functionality to create or modify zip archives. It supports: + +- Zipping Files and Directories: You can zip both individual files and entire directories. +- Appending to Existing Archives: Files can be appended to an existing zip archive. +- Exclude Patterns (-x): You can specify files or patterns to exclude while zipping. +- Include Patterns (-i): You can include specific files or patterns while zipping. +- Delete Patterns (-d): You can delete specific files from an existing zip archive. +- PreserveMtimes: Controls whether to preserve the last modification timestamps (mtimes) of files included in the archive. + 1. true (default): Maintains the original mtimes. + 2. false: Sets the mtime of all files in the archive to the current time during creation. + +===== Zipping Files and Folders +[source,scala] +---- +val pathsToZip = List( + os.Path("/path/to/file1.txt"), + os.Path("/path/to/folder") +) +os.zip(os.Path("/path/to/destination.zip"), pathsToZip) +---- + +This will create a new zip archive at `/path/to/destination.zip` containing `file1.txt` and everything inside folder. + +===== Appending to an Existing Zip File +[source,scala] +---- +os.zip( + os.Path("/path/to/existing.zip"), + List(os.Path("/path/to/newfile.txt")), + appendToExisting = true +) +---- + +This will append `newfile.txt` to the existing zip file existing.zip. + +===== Excluding/Including Files in Zip + +You can specify files or folders to be excluded or included when creating the zip: + + +[source,scala] +---- +os.zip( + os.Path("/path/to/destination.zip"), + List(os.Path("/path/to/folder")), + excludePatterns = List(".*\\.log", "temp/.*"), // Exclude log files and "temp" folder + includePatterns = List(".*\\.txt") // Include only .txt files +) + +---- + +This will include only `.txt` files, excluding any `.log` files and anything inside the `temp` folder. + +===== Streaming Zip Operation + +The `os` library offers additional method, `os.zip.stream` that allows you to work with zip data in a streaming fashion. This is useful when dealing with large archives or when you need to write the zipped data to a specific output stream. + +---- +os.zip.stream( + os.Path("/path/to/destination.zip"), + List(os.Path("/path/to/folder")), + excludePatterns = List(".*\\.log", "temp/.*"), // Exclude log files and "temp" folder + includePatterns = List(".*\\.txt") // Include only .txt files +) +---- + +==== `os.unzip` + +===== Unzipping Files +[source,scala] +---- +os.unzip(os.Path("/path/to/archive.zip"), Some(os.Path("/path/to/destination"))) +---- + +This extracts the contents of `archive.zip` to the specified destination. + + +===== Excluding Files While Unzipping +You can exclude certain files from being extracted using patterns: + +[source,scala] +---- +os.unzip( + os.Path("/path/to/archive.zip"), + Some(os.Path("/path/to/destination")), + excludePatterns = List(".*\\.log", "temp/.*") // Exclude log files and the "temp" folder +) +---- + +===== Listing Archive Contents +You can list the contents of the zip file without extracting them: + +[source,scala] +---- +os.unzip(os.Path("/path/to/archive.zip"), listOnly = true) +---- + +This will print all the file paths contained in the zip archive. + +===== Streaming UnZip Operation + +This function takes a `geny.Readable` object representing the zip data stream and a destination directory path. It extracts the contents of the zip to the specified destination while allowing you to exclude certain files based on patterns. + +---- +os.unzip.stream( + os.Path("/path/to/destination.zip"), + List(os.Path("/path/to/folder")), + excludePatterns = List(".*\\.log", "temp/.*") // Exclude log files and "temp" folder +) +---- + === Filesystem Metadata ==== `os.stat` @@ -1708,13 +1835,13 @@ val yes10 = os.proc("yes") ---- This feature is implemented inside the library and will terminate any process reading the -stdin of other process in pipeline on every IO error. This behavior can be disabled via the -`handleBrokenPipe` flag on `call` and `spawn` methods. Note that Windows does not support -broken pipe behaviour, so a command like`yes` would run forever. `handleBrokenPipe` is set +stdin of other process in pipeline on every IO error. This behavior can be disabled via the +`handleBrokenPipe` flag on `call` and `spawn` methods. Note that Windows does not support +broken pipe behaviour, so a command like`yes` would run forever. `handleBrokenPipe` is set to false by default on Windows. Both `call` and `spawn` correspond in their behavior to their counterparts in the `os.proc`, -but `spawn` returns the `os.ProcessPipeline` instance instead. It offers the same +but `spawn` returns the `os.ProcessPipeline` instance instead. It offers the same `API` as `SubProcess`, but will operate on the set of processes instead of a single one. `Pipefail` is enabled by default, so if any of the processes in the pipeline fails, the whole @@ -2105,8 +2232,8 @@ explicitly choose to convert relative paths to absolute using some base. ==== Roots and filesystems -If you are using a system that supports different roots of paths, e.g. Windows, -you can use the argument of `os.root` to specify which root you want to use. +If you are using a system that supports different roots of paths, e.g. Windows, +you can use the argument of `os.root` to specify which root you want to use. If not specified, the default root will be used (usually, C on Windows, / on Unix). [source,scala] @@ -2128,11 +2255,11 @@ val fs = FileSystems.newFileSystem(uri, env); val path = os.root("/", fs) / "dir" ---- -Note that the jar file system operations suchs as writing to a file are supported -only on JVM 11+. Depending on the filesystem, some operations may not be supported - -for example, running an `os.proc` with pwd in a jar file won't work. You may also -meet limitations imposed by the implementations - in jar file system, the files are -created only after the file system is closed. Until that, the ones created in your +Note that the jar file system operations suchs as writing to a file are supported +only on JVM 11+. Depending on the filesystem, some operations may not be supported - +for example, running an `os.proc` with pwd in a jar file won't work. You may also +meet limitations imposed by the implementations - in jar file system, the files are +created only after the file system is closed. Until that, the ones created in your program are kept in memory. ==== `os.ResourcePath` @@ -2199,9 +2326,9 @@ By default, the following types of values can be used where-ever ``os.Source``s are required: * Any `geny.Writable` data type: - ** `Array[Byte]` - ** `java.lang.String` (these are treated as UTF-8) - ** `java.io.InputStream` +** `Array[Byte]` +** `java.lang.String` (these are treated as UTF-8) +** `java.io.InputStream` * `java.nio.channels.SeekableByteChannel` * Any `TraversableOnce[T]` of the above: e.g. `Seq[String]`, `List[Array[Byte]]`, etc. @@ -2266,9 +2393,9 @@ string, int or set representations of the `os.PermSet` via: === 0.10.7 * Allow multi-segment paths segments for literals https://github.com/com-lihaoyi/os-lib/pull/297: You - can now write `os.pwd / "foo/bar/qux"` rather than `os.pwd / "foo" / "bar" / "qux"`. Note that this - is only allowed for string literals, and non-literal path segments still need to be wrapped e.g. - `def myString = "foo/bar/qux"; os.pwd / os.SubPath(myString)` for security and safety purposes +can now write `os.pwd / "foo/bar/qux"` rather than `os.pwd / "foo" / "bar" / "qux"`. Note that this +is only allowed for string literals, and non-literal path segments still need to be wrapped e.g. +`def myString = "foo/bar/qux"; os.pwd / os.SubPath(myString)` for security and safety purposes [#0-10-6] === 0.10.6 @@ -2279,23 +2406,23 @@ string, int or set representations of the `os.PermSet` via: === 0.10.5 * Introduce `os.SubProcess.env` `DynamicVariable` to override default `env` - (https://github.com/com-lihaoyi/os-lib/pull/295) +(https://github.com/com-lihaoyi/os-lib/pull/295) [#0-10-4] === 0.10.4 * Add a lightweight syntax for `os.call()` and `os.spawn` APIs - (https://github.com/com-lihaoyi/os-lib/pull/292) +(https://github.com/com-lihaoyi/os-lib/pull/292) * Add a configurable grace period when subprocesses timeout and have to - be terminated to give a chance for shutdown logic to run - (https://github.com/com-lihaoyi/os-lib/pull/286) +be terminated to give a chance for shutdown logic to run +(https://github.com/com-lihaoyi/os-lib/pull/286) [#0-10-3] === 0.10.3 * `os.Inherit` now can be redirected on a threadlocal basis via `os.Inherit.in`, `.out`, or `.err`. - `os.InheritRaw` is available if you do not want the redirects to take effect +`os.InheritRaw` is available if you do not want the redirects to take effect [#0-10-2] diff --git a/os/src/ZipOps.scala b/os/src/ZipOps.scala new file mode 100644 index 00000000..34cd503e --- /dev/null +++ b/os/src/ZipOps.scala @@ -0,0 +1,564 @@ +package os + +import geny.{Readable, Writable} + +import java.io._ +import java.nio.file.attribute.{ + BasicFileAttributeView, + FileTime, + PosixFileAttributeView, + PosixFilePermissions +} +import java.nio.file.{Files, Path, StandardCopyOption} +import java.util.zip.{ZipEntry, ZipFile, ZipInputStream, ZipOutputStream} +import scala.collection.JavaConverters._ +import scala.util.matching.Regex + +object zip { + + /** + * Zips the provided list of files and directories into a single ZIP archive. + * + * @param destination The path to the destination ZIP file. + * @param listOfPaths A list of paths to files and directories to be zipped. Defaults to an empty list. + * @param appendToExisting Whether to append the listed paths to an existing ZIP file (if it exists). Defaults to false. + * @param excludePatterns A list of regular expression patterns to exclude files from the ZIP archive. Defaults to an empty list. + * @param includePatterns A list of regular expression patterns to include files in the ZIP archive. Defaults to an empty list (includes all files). + * @param deletePatterns A list of regular expression patterns to delete files from an existing ZIP archive before appending new ones. Defaults to an empty list. + * @param preserveMtimes Whether to preserve modification times (mtimes) of the files. + * @param preservePerms Whether to preserve file permissions (POSIX). + * @return The path to the created ZIP archive. + */ + def apply( + destination: os.Path, + listOfPaths: List[os.Path] = List(), + appendToExisting: Boolean = false, + excludePatterns: List[String] = List(), // -x option + includePatterns: List[String] = List(), // -i option + deletePatterns: List[String] = List(), // -d option + preserveMtimes: Boolean = true, + preservePermissions: Boolean = true + ): os.Path = { + + val javaNIODestination: java.nio.file.Path = destination.toNIO + val pathsToBeZipped: List[java.nio.file.Path] = listOfPaths.map(_.toNIO) + + // Convert the string patterns into regex + val excludeRegexPatterns: List[Regex] = excludePatterns.map(_.r) + val includeRegexPatterns: List[Regex] = includePatterns.map(_.r) + val deleteRegexPatterns: List[Regex] = deletePatterns.map(_.r) + + val zipFilePath: java.nio.file.Path = resolveDestinationZipFile(javaNIODestination) + + if (Files.exists(zipFilePath) && deletePatterns.nonEmpty) { + deleteFilesFromZip(zipFilePath, deleteRegexPatterns) + } else { + if (appendToExisting && Files.exists(zipFilePath)) { + appendToExistingZip( + zipFilePath, + pathsToBeZipped, + excludeRegexPatterns, + includeRegexPatterns, + preserveMtimes, + preservePermissions + ) + } else { + createNewZip( + zipFilePath, + pathsToBeZipped, + excludeRegexPatterns, + includeRegexPatterns, + preserveMtimes, + preservePermissions + ) + } + + } + os.Path(zipFilePath) + } + + private def createNewZip( + zipFilePath: java.nio.file.Path, + pathsToBeZipped: List[java.nio.file.Path], + excludePatterns: List[Regex], + includePatterns: List[Regex], + preserveMtimes: Boolean, + preservePerms: Boolean + ): Unit = { + val zipOut = new ZipOutputStream(new FileOutputStream(zipFilePath.toFile)) + try { + pathsToBeZipped.foreach { path => + val file = path.toFile + if (shouldInclude(file.getName, excludePatterns, includePatterns)) { + if (file.isDirectory) { + zipFolder( + file, + file.getName, + zipOut, + excludePatterns, + includePatterns, + preserveMtimes, + preservePerms + ) + } else { + zipFile(file, zipOut, preserveMtimes, preservePerms) + } + } + } + } finally { + zipOut.close() + } + } + + private def appendToExistingZip( + zipFilePath: java.nio.file.Path, + pathsToBeZipped: List[java.nio.file.Path], + excludePatterns: List[Regex], + includePatterns: List[Regex], + preserveMtimes: Boolean, + preservePerms: Boolean + ): Unit = { + val tempOut = new ByteArrayOutputStream() + val zipOut = new ZipOutputStream(tempOut) + + val existingZip = new ZipFile(zipFilePath.toFile) + + existingZip.entries().asScala.foreach { entry => + if (shouldInclude(entry.getName, excludePatterns, includePatterns)) { + val inputStream = existingZip.getInputStream(entry) + zipOut.putNextEntry(new ZipEntry(entry.getName)) + + val buffer = new Array[Byte](1024) + var length = inputStream.read(buffer) + while (length > 0) { + zipOut.write(buffer, 0, length) + length = inputStream.read(buffer) + } + inputStream.close() + zipOut.closeEntry() + } + } + + pathsToBeZipped.foreach { path => + val file = path.toFile + if (shouldInclude(file.getName, excludePatterns, includePatterns)) { + if (file.isDirectory) { + zipFolder( + file, + file.getName, + zipOut, + excludePatterns, + includePatterns, + preserveMtimes, + preservePerms + ) + } else { + zipFile(file, zipOut, preserveMtimes, preservePerms) + } + } + } + + zipOut.close() + + val newZipContent = tempOut.toByteArray + Files.write(zipFilePath, newZipContent) + } + + private def deleteFilesFromZip( + zipFilePath: java.nio.file.Path, + deletePatterns: List[Regex] + ): Unit = { + val tempOut = new ByteArrayOutputStream() + val zipOut = new ZipOutputStream(tempOut) + + val existingZip = new ZipFile(zipFilePath.toFile) + + existingZip.entries().asScala.foreach { entry => + if (!deletePatterns.exists(_.findFirstIn(entry.getName).isDefined)) { + val inputStream = existingZip.getInputStream(entry) + zipOut.putNextEntry(new ZipEntry(entry.getName)) + + val buffer = new Array[Byte](1024) + var length = inputStream.read(buffer) + while (length > 0) { + zipOut.write(buffer, 0, length) + length = inputStream.read(buffer) + } + inputStream.close() + zipOut.closeEntry() + } + } + + zipOut.close() + + val newZipContent = tempOut.toByteArray + Files.write(zipFilePath, newZipContent) + } + + private def shouldInclude( + fileName: String, + excludePatterns: List[Regex], + includePatterns: List[Regex] + ): Boolean = { + val isExcluded = excludePatterns.exists(_.findFirstIn(fileName).isDefined) + val isIncluded = + includePatterns.isEmpty || includePatterns.exists(_.findFirstIn(fileName).isDefined) + !isExcluded && isIncluded + } + + private def zipFile( + file: java.io.File, + zipOut: ZipOutputStream, + preserveMtimes: Boolean, + preservePerms: Boolean + ): Unit = { + val fis = new FileInputStream(file) + val zipEntry = new ZipEntry(file.getName) + + // Preserve modification time if requested + if (preserveMtimes) { + zipEntry.setTime(file.lastModified()) + } + + // Set permissions if requested and supported + val posixView = Files.getFileAttributeView(file.toPath, classOf[PosixFileAttributeView]) + if (preservePerms && posixView != null) { + val attrs = posixView.readAttributes() + val perms = attrs.permissions() + zipEntry.setComment(PosixFilePermissions.toString(perms)) + } + + zipOut.putNextEntry(zipEntry) + + val buffer = new Array[Byte](1024) + var length = fis.read(buffer) + while (length >= 0) { + zipOut.write(buffer, 0, length) + length = fis.read(buffer) + } + + fis.close() + } + + private def zipFolder( + folder: java.io.File, + parentFolderName: String, + zipOut: ZipOutputStream, + excludePatterns: List[Regex], + includePatterns: List[Regex], + preserveMtimes: Boolean, + preservePerms: Boolean + ): Unit = { + val files = folder.listFiles() + if (files != null) { + files.foreach { file => + if (shouldInclude(file.getName, excludePatterns, includePatterns)) { + if (file.isDirectory) { + zipFolder( + file, + parentFolderName + "/" + file.getName, + zipOut, + excludePatterns, + includePatterns, + preserveMtimes, + preservePerms + ) + } else { + val fis = new FileInputStream(file) + val zipEntry = new ZipEntry(parentFolderName + "/" + file.getName) + + // Preserve modification time if requested + if (preserveMtimes) { + zipEntry.setTime(file.lastModified()) + } + + // Set permissions if requested and supported + val posixView = Files.getFileAttributeView(file.toPath, classOf[PosixFileAttributeView]) + if (preservePerms && posixView != null) { + val attrs = posixView.readAttributes() + val perms = attrs.permissions() + zipEntry.setComment(PosixFilePermissions.toString(perms)) + } + + zipOut.putNextEntry(zipEntry) + + val buffer = new Array[Byte](1024) + var length = fis.read(buffer) + while (length >= 0) { + zipOut.write(buffer, 0, length) + length = fis.read(buffer) + } + + fis.close() + } + } + } + } + } + + private def resolveDestinationZipFile(destination: java.nio.file.Path): java.nio.file.Path = { + val zipFilePath: java.nio.file.Path = if (Files.isDirectory(destination)) { + destination.resolve("archive.zip") + } else { + destination + } + zipFilePath + } + + /** + * Zips a folder recursively and returns a geny.Writable for streaming the ZIP data. + * + * @param source The path to the folder to be zipped. + * @param destination The path to the destination ZIP file (optional). If not provided, a temporary ZIP file will be created. + * @param appendToExisting Whether to append the listed paths to an existing ZIP file (if it exists). Defaults to false. + * @param excludePatterns A list of regular expression patterns to exclude files during zipping. Defaults to an empty list. + * @param includePatterns A list of regular expression patterns to include files in the ZIP archive. Defaults to an empty list (includes all files). + * @param deletePatterns A list of regular expression patterns to delete files from an existing ZIP archive before appending new ones. Defaults to an empty list. + * @param preserveMtimes Whether to preserve modification times (mtimes) of the files. + * @param preservePerms Whether to preserve file permissions (POSIX). + * @return A geny.Writable object for writing the ZIP data. + */ + def stream( + source: os.Path, + destination: Option[os.Path] = None, + appendToExisting: Boolean = false, + excludePatterns: List[String] = List(), + includePatterns: List[String] = List(), + deletePatterns: List[String] = List(), + preserveMtimes: Boolean = false, // Preserve modification times + preservePerms: Boolean = false // Preserve POSIX permissions + ): geny.Writable = { + + val zipOut = new ByteArrayOutputStream() + val zipOutputStream = new ZipOutputStream(zipOut) + + try { + val zipFilePath = destination.getOrElse { + val tempFile = File.createTempFile( + "temp-archive", + ".zip", + new File(System.getProperty("java.io.tmpdir")) + ) + os.Path(tempFile) + } + val javaNIODestination: java.nio.file.Path = zipFilePath.toNIO + val pathsToBeZipped = List(source.toNIO) + + // Convert the string patterns into regex + val excludeRegexPatterns: List[Regex] = excludePatterns.map(_.r) + val includeRegexPatterns: List[Regex] = includePatterns.map(_.r) + val deleteRegexPatterns: List[Regex] = deletePatterns.map(_.r) + + if (Files.exists(javaNIODestination) && deletePatterns.nonEmpty) { + deleteFilesFromZip(javaNIODestination, deleteRegexPatterns) + } else { + if (appendToExisting && Files.exists(javaNIODestination)) { + appendToExistingZip( + javaNIODestination, + pathsToBeZipped, + excludeRegexPatterns, + includeRegexPatterns, + preserveMtimes, + preservePerms + ) + } else { + createNewZip( + javaNIODestination, + pathsToBeZipped, + excludeRegexPatterns, + includeRegexPatterns, + preserveMtimes, + preservePerms + ) + } + } + } finally { + zipOutputStream.close() + } + + (outputStream: OutputStream) => { + zipOut.writeTo(outputStream) + } + } +} + +object unzip { + + /** + * Unzips a ZIP archive to a specified destination directory. + * + * @param source The path to the ZIP archive. + * @param destination (Optional) The path to the destination directory for extracted files. + * If not provided, the source directory will be used. + * @param excludePatterns (Optional) A list of regular expression patterns to exclude files from extraction. + * @param listOnly (Optional) If `true`, lists the contents of the ZIP archive without extracting them. + * @return The path to the destination directory where files are extracted, or the source path if `listOnly` is `true`. + * + * This function extracts the contents of the specified ZIP archive (`source`) to the given destination directory (`destination`). + * It allows you to control which files are extracted by providing a list of regular expression patterns (`excludePatterns`) + * to exclude specific files or folders. You can also use the `listOnly` flag to get a listing of the ZIP contents without actually extracting them. + * + * If no `destination` is provided, the ZIP archive will be extracted to the same directory as the source file. + * + * @throws java.io.IOException If there's an error while accessing or extracting the ZIP archive. + */ + def apply( + source: os.Path, + destination: Option[os.Path] = None, + excludePatterns: List[String] = List(), // Patterns to exclude + listOnly: Boolean = false // List contents without extracting + ): os.Path = { + + val sourcePath: java.nio.file.Path = source.toNIO + + // Ensure the source file is a zip file + validateZipFile(sourcePath) + + // Convert the exclusion patterns to regex + val excludeRegexPatterns: List[Regex] = excludePatterns.map(_.r) + + // If listOnly is true, list the contents and return the source path + if (listOnly) { + listContents(sourcePath, excludeRegexPatterns) + return source + } + + // Determine the destination directory + val destPath = destination match { + case Some(dest) => createDestinationDirectory(dest) + case None => sourcePath.getParent // Unzip in the same directory as the source + } + + // Perform the unzip operation + unzipFile(sourcePath, destPath, excludeRegexPatterns) + os.Path(destPath) + } + + /** Validates if the input file is a valid zip file */ + private def validateZipFile(sourcePath: java.nio.file.Path): Unit = { + if (!Files.exists(sourcePath)) { + throw new IllegalArgumentException(s"Source file does not exist: $sourcePath") + } + if (!sourcePath.toString.endsWith(".zip")) { + throw new IllegalArgumentException(s"Source file is not a zip file: $sourcePath") + } + } + + /** Creates the destination directory if it doesn't exist */ + private def createDestinationDirectory(destination: os.Path): java.nio.file.Path = { + val destPath: java.nio.file.Path = destination.toNIO + if (!Files.exists(destPath)) { + Files.createDirectories(destPath) // Create directory if absent + } + destPath + } + + private def unzipFile( + sourcePath: java.nio.file.Path, + destPath: java.nio.file.Path, + excludePatterns: List[Regex] + ): Unit = { + + val zipInputStream = new ZipInputStream(new FileInputStream(sourcePath.toFile)) + + try { + var zipEntry: ZipEntry = zipInputStream.getNextEntry + while (zipEntry != null) { + // Skip files that match the exclusion patterns + if (!shouldExclude(zipEntry.getName, excludePatterns)) { + val newFile = createFileForEntry(destPath, zipEntry) + if (zipEntry.isDirectory) { + Files.createDirectories(newFile.toPath) + } else { + extractFile(zipInputStream, newFile) + } + } + zipEntry = zipInputStream.getNextEntry + } + } finally { + zipInputStream.closeEntry() + zipInputStream.close() + } + } + + /** Lists the contents of the zip file */ + private def listContents(sourcePath: java.nio.file.Path, excludePatterns: List[Regex]): Unit = { + val zipFile = new ZipFile(sourcePath.toFile) + + zipFile.entries().asScala.foreach { entry => + if (!shouldExclude(entry.getName, excludePatterns)) { + println(entry.getName) + } + } + } + + /** Creates the file for the current ZipEntry, preserving the directory structure */ + private def createFileForEntry(destDir: java.nio.file.Path, zipEntry: ZipEntry): File = { + val newFile = new File(destDir.toFile, zipEntry.getName) + val destDirPath = destDir.toFile.getCanonicalPath + val newFilePath = newFile.getCanonicalPath + + // Ensure that the file path is within the destination directory to avoid Zip Slip vulnerability + if (!newFilePath.startsWith(destDirPath)) { + throw new SecurityException(s"Entry is outside of the target directory: ${zipEntry.getName}") + } + newFile + } + + /** Extracts the file content from the zip to the destination */ + private def extractFile(zipInputStream: ZipInputStream, file: File): Unit = { + val parentDir = file.getParentFile + if (!parentDir.exists()) { + parentDir.mkdirs() // Ensure parent directories exist + } + + val outputStream = Files.newOutputStream(file.toPath) + val buffer = new Array[Byte](1024) + var len = zipInputStream.read(buffer) + while (len > 0) { + outputStream.write(buffer, 0, len) + len = zipInputStream.read(buffer) + } + outputStream.close() + } + + /** Determines if a file should be excluded based on the given patterns */ + def shouldExclude(fileName: String, excludePatterns: List[Regex]): Boolean = { + excludePatterns.exists(_.findFirstIn(fileName).isDefined) + } + + /** + * Unzips a ZIP data stream represented by a geny.Readable and extracts it to a destination directory. + * + * @param source A geny.Readable object representing the ZIP data stream. + * @param destination The path to the destination directory for extracted files. + * @param excludePatterns A list of regular expression patterns to exclude files during extraction. (Optional) + */ + def stream( + source: geny.Readable, + destination: os.Path, + excludePatterns: List[String] = List() + ): Unit = { + source.readBytesThrough { inputStream => + val zipInputStream = new ZipInputStream(inputStream) + try { + var zipEntry: ZipEntry = zipInputStream.getNextEntry + while (zipEntry != null) { + // Skip files that match the exclusion patterns + if (!shouldExclude(zipEntry.getName, excludePatterns.map(_.r))) { + val newFile = createFileForEntry(destination.toNIO, zipEntry) + if (zipEntry.isDirectory) { + Files.createDirectories(newFile.toPath) + } else { + extractFile(zipInputStream, newFile) + } + } + zipEntry = zipInputStream.getNextEntry + } + } finally { + zipInputStream.closeEntry() + zipInputStream.close() + } + } + } +} diff --git a/os/test/src/ZipOpTests.scala b/os/test/src/ZipOpTests.scala new file mode 100644 index 00000000..87e7fc3e --- /dev/null +++ b/os/test/src/ZipOpTests.scala @@ -0,0 +1,306 @@ +package test.os + +import os.zip +import test.os.TestUtil.prep +import utest._ + +import java.io.{ByteArrayInputStream, ByteArrayOutputStream, PrintStream, PrintWriter, StringWriter} +import java.util.zip.{ZipEntry, ZipFile, ZipInputStream, ZipOutputStream} +import scala.collection.immutable.Seq +import java.nio.file.{Files, Paths} +import java.nio.file.attribute.FileTime +import java.time.{Instant, ZoneId} +import java.time.temporal.{ChronoUnit, TemporalUnit} +import scala.collection.JavaConverters._ + +object ZipOpTests extends TestSuite { + + def tests = Tests { + + test("zipAndUnzipFolder") { + + test - prep { wd => + // Zipping files and folders in a new zip file + val zipFileName = "zip-file-test.zip" + val zipFile1: os.Path = os.zip( + destination = wd / zipFileName, + listOfPaths = List( + wd / "File.txt", + wd / "folder1" + ) + ) + + // Adding files and folders to an existing zip file + val zipFile2: os.Path = os.zip( + destination = wd / zipFileName, + listOfPaths = List( + wd / "folder2", + wd / "Multi Line.txt" + ), + appendToExisting = true + ) + + // Unzip file to a destination folder + val unzippedFolder = os.unzip( + source = wd / zipFileName, + destination = Some(wd / "unzipped folder") + ) + + val paths = os.walk(unzippedFolder) + assert(paths.length == 9) + assert(paths.contains(unzippedFolder / "File.txt")) + assert(paths.contains(unzippedFolder / "Multi Line.txt")) + assert(paths.contains(unzippedFolder / "folder1" / "one.txt")) + assert(paths.contains(unzippedFolder / "folder2" / "nestedB" / "b.txt")) + } + } + + test("zipByExcludingCertainFiles") { + + test - prep { wd => + val amxFile = "File.amx" + os.copy(wd / "File.txt", wd / amxFile) + + // Zipping files and folders in a new zip file + val zipFileName = "zipByExcludingCertainFiles.zip" + val zipFile1: os.Path = os.zip( + destination = wd / zipFileName, + listOfPaths = List( + wd / "File.txt", + wd / amxFile, + wd / "Multi Line.txt" + ), + excludePatterns = List(".*\\.txt") + ) + + // Unzip file to check for contents + val outputZipFilePath = + os.unzip(zipFile1, destination = Some(wd / "zipByExcludingCertainFiles")) + val paths = os.walk(outputZipFilePath) + assert(paths.length == 1) + assert(paths.contains(outputZipFilePath / amxFile)) + } + } + + test("zipByIncludingCertainFiles") { + + test - prep { wd => + val amxFile = "File.amx" + os.copy(wd / "File.txt", wd / amxFile) + + // Zipping files and folders in a new zip file + val zipFileName = "zipByIncludingCertainFiles.zip" + val zipFile1: os.Path = os.zip( + destination = wd / zipFileName, + listOfPaths = List( + wd / "File.txt", + wd / amxFile, + wd / "Multi Line.txt" + ), + includePatterns = List(".*\\.amx") + ) + + // Unzip file to check for contents + val outputZipFilePath = + os.unzip(zipFile1, destination = Some(wd / "zipByIncludingCertainFiles")) + val paths = os.walk(outputZipFilePath) + assert(paths.length == 1) + assert(paths.contains(outputZipFilePath / amxFile)) + } + } + + test("zipByDeletingCertainFiles") { + + test - prep { wd => + val amxFile = "File.amx" + os.copy(wd / "File.txt", wd / amxFile) + + // Zipping files and folders in a new zip file + val zipFileName = "zipByDeletingCertainFiles.zip" + val zipFile1: os.Path = os.zip( + destination = wd / zipFileName, + listOfPaths = List( + wd / "File.txt", + wd / amxFile, + wd / "Multi Line.txt" + ) + ) + + os.zip( + destination = zipFile1, + deletePatterns = List(amxFile) + ) + + // Unzip file to check for contents + val outputZipFilePath = + os.unzip(zipFile1, destination = Some(wd / "zipByDeletingCertainFiles")) + val paths = os.walk(outputZipFilePath) + assert(paths.length == 2) + assert(paths.contains(outputZipFilePath / "File.txt")) + assert(paths.contains(outputZipFilePath / "Multi Line.txt")) + } + } + + test("listContentsOfZipFileWithoutExtracting") { + + test - prep { wd => + // Zipping files and folders in a new zip file + val zipFileName = "listContentsOfZipFileWithoutExtracting.zip" + val zipFile: os.Path = os.zip( + destination = wd / zipFileName, + listOfPaths = List( + wd / "File.txt", + wd / "folder1" + ) + ) + val originalOut = System.out + val outputStream = new ByteArrayOutputStream() + System.setOut(new PrintStream(outputStream)) + + // Unzip file to a destination folder + val unzippedFolder = os.unzip( + source = wd / zipFileName, + listOnly = true + ) + + // Then + val capturedOutput: Array[String] = outputStream.toString.split("\n") + assert(capturedOutput.length <= 2) + + // Restore the original output stream + System.setOut(originalOut) + } + } + + test("unzipAllExceptExcludingCertainFiles") { + + test - prep { wd => + val amxFile = "File.amx" + os.copy(wd / "File.txt", wd / amxFile) + + val zipFileName = "unzipAllExceptExcludingCertainFiles.zip" + val zipFile: os.Path = os.zip( + destination = wd / zipFileName, + listOfPaths = List( + wd / "File.txt", + wd / amxFile, + wd / "folder1" + ) + ) + + // Unzip file to a destination folder + val unzippedFolder = os.unzip( + source = wd / zipFileName, + destination = Some(wd / "unzipAllExceptExcludingCertainFiles"), + excludePatterns = List(amxFile) + ) + + val paths = os.walk(unzippedFolder) + assert(paths.length == 3) + assert(paths.contains(unzippedFolder / "File.txt")) + assert(paths.contains(unzippedFolder / "folder1" / "one.txt")) + } + } + + test("zipStreamFunction") { + test - prep { wd => + val streamOutput = new ByteArrayOutputStream() + val zipFileName = "zipStreamFunction.zip" + + // Create a stream to zip a folder + val writable = zip.stream( + source = wd / "File.txt", + destination = Some(wd / zipFileName), + excludePatterns = List(), + includePatterns = List(), + deletePatterns = List() + ) + + // Write the zipped data to the stream + writable.writeBytesTo(streamOutput) + + val unzippedFolder = os.unzip( + source = wd / zipFileName, + destination = Some(wd / "zipStreamFunction") + ) + val paths = os.walk(unzippedFolder) + assert(paths.length == 1) + assert(paths.contains(unzippedFolder / "File.txt")) + } + } + + test("unzipStreamFunction") { + test - prep { wd => + // Step 1: Create an in-memory ZIP file as a stream + val zipStreamOutput = new ByteArrayOutputStream() + val zipOutputStream = new ZipOutputStream(zipStreamOutput) + + // Step 2: Add some files to the ZIP + val file1Name = "file1.txt" + val file2Name = "nested/folder/file2.txt" + + // Add first file + zipOutputStream.putNextEntry(new ZipEntry(file1Name)) + zipOutputStream.write("Content of file1".getBytes) + zipOutputStream.closeEntry() + + // Add second file inside a nested folder + zipOutputStream.putNextEntry(new ZipEntry(file2Name)) + zipOutputStream.write("Content of file2".getBytes) + zipOutputStream.closeEntry() + + // Close the ZIP output stream + zipOutputStream.close() + + // Step 3: Prepare the destination folder for unzipping + val unzippedFolder = wd / "unzipped-stream-folder" + val readableZipStream = geny.Readable.ByteArrayReadable(zipStreamOutput.toByteArray) + + // Unzipping the stream to the destination folder + os.unzip.stream( + source = readableZipStream, + destination = unzippedFolder + ) + + // Step 5: Verify the unzipped files and contents + val paths = os.walk(unzippedFolder) + assert(paths.contains(unzippedFolder / file1Name)) + assert(paths.contains(unzippedFolder / "nested" / "folder" / "file2.txt")) + + // Check the contents of the files + val file1Content = os.read(unzippedFolder / file1Name) + val file2Content = os.read(unzippedFolder / "nested" / "folder" / "file2.txt") + + assert(file1Content == "Content of file1") + assert(file2Content == "Content of file2") + } + } + + test("zipAndUnzipPreserveMtimes") { + test - prep { wd => + // Create a file and set its modification time + val testFile = wd / "FileWithMtime.txt" + os.write(testFile, "Test content") + + // Use basic System.currentTimeMillis() for modification time + val originalMtime = System.currentTimeMillis() - (1 * 60 * 1000) // 1 minute ago + val path = Paths.get(testFile.toString) + Files.setLastModifiedTime(path, FileTime.fromMillis(originalMtime)) + + // Zipping the file with preserveMtimes = true + val zipFileName = "zipWithMtimePreservation.zip" + val zipFile: os.Path = os.zip( + destination = wd / zipFileName, + listOfPaths = List(testFile), + preserveMtimes = true + ) + + val existingZipFile = new ZipFile(zipFile.toNIO.toFile) + val actualMTime = existingZipFile.entries().asScala.toList.head.getTime + + // Compare the original and actual modification times (in minutes) + assert((originalMtime / (1000 * 60)) == (actualMTime / (1000 * 60))) + } + } + } +}