From 6058343af84e84e6faddfd6a91483ba1f7db7941 Mon Sep 17 00:00:00 2001 From: Chaitanya Waikar Date: Sun, 29 Sep 2024 13:22:52 +0200 Subject: [PATCH 01/15] Introduce `zipIn` functionality to zip files and folders in a new zip file. The zipIn functionality will create a new file `archive.zip` if the name of the zip file is absent --- os/src/ZipOps.scala | 93 ++++++++++++++++++++++++++++++++++++ os/test/src/ZipOpTests.scala | 39 +++++++++++++++ 2 files changed, 132 insertions(+) create mode 100644 os/src/ZipOps.scala create mode 100644 os/test/src/ZipOpTests.scala diff --git a/os/src/ZipOps.scala b/os/src/ZipOps.scala new file mode 100644 index 00000000..332394db --- /dev/null +++ b/os/src/ZipOps.scala @@ -0,0 +1,93 @@ +package os + +import java.io.{FileInputStream, FileOutputStream} +import java.net.URI +import java.nio.file.{Files, Path, StandardCopyOption} +import java.util.zip.{ZipEntry, ZipOutputStream} + +object zipIn { + + def apply( + destination: os.Path, + listOfPaths: List[os.Path], + options: List[String] = List.empty + ): os.Path = { + val javaNIODestination: java.nio.file.Path = destination.toNIO + val pathsToBeZipped: List[java.nio.file.Path] = listOfPaths.map(_.toNIO) + + val zipFilePath: java.nio.file.Path = resolveDestinationZipFile(javaNIODestination) + + // Create a zip output stream + val zipOut = new ZipOutputStream(new FileOutputStream(zipFilePath.toFile)) + + try { + pathsToBeZipped.foreach { path => + val file = path.toFile + if (file.isDirectory) { + // Zip the folder recursively + zipFolder(file, file.getName, zipOut) + } else { + // Zip the individual file + zipFile(file, zipOut) + } + } + } finally { + zipOut.close() + } + + os.Path(zipFilePath) // Return the path to the created zip file + } + + private def zipFile(file: java.io.File, zipOut: ZipOutputStream): Unit = { + val fis = new FileInputStream(file) + val zipEntry = new ZipEntry(file.getName) + zipOut.putNextEntry(zipEntry) + + // Define a 1KB buffer for reading file content + val buffer = new Array[Byte](1024) + var length = fis.read(buffer) + while (length >= 0) { + zipOut.write(buffer, 0, length) + length = fis.read(buffer) + } + + fis.close() + } + + private def zipFolder(folder: java.io.File, parentFolderName: String, zipOut: ZipOutputStream): Unit = { + val files = folder.listFiles() + if (files != null) { + files.foreach { file => + if (file.isDirectory) { + // Recursively zip subdirectory + zipFolder(file, parentFolderName + "/" + file.getName, zipOut) + } else { + // Add the file with its relative path + val fis = new FileInputStream(file) + val zipEntry = new ZipEntry(parentFolderName + "/" + file.getName) + zipOut.putNextEntry(zipEntry) + + // Write file content to zip + val buffer = new Array[Byte](1024) + var length = fis.read(buffer) + while (length >= 0) { + zipOut.write(buffer, 0, length) + length = fis.read(buffer) + } + + fis.close() + } + } + } + } + + private def resolveDestinationZipFile(destination: java.nio.file.Path): java.nio.file.Path = { + // Check if destination is a directory or file + val zipFilePath: java.nio.file.Path = if (Files.isDirectory(destination)) { + destination.resolve("archive.zip") // Append default zip name + } else { + destination // Use provided file name + } + zipFilePath + } +} diff --git a/os/test/src/ZipOpTests.scala b/os/test/src/ZipOpTests.scala new file mode 100644 index 00000000..c8a7a56c --- /dev/null +++ b/os/test/src/ZipOpTests.scala @@ -0,0 +1,39 @@ +package test.os + +import test.os.TestUtil.prep +import utest._ + +import scala.collection.immutable.Seq + +object ZipOpTests extends TestSuite { + + def tests = Tests { + + test("zipOps") { + + test - prep { wd => + // Zipping files and folders in a new zip file + val zipFile1: os.Path = os.zipIn( + destination = wd / "zip-file1-test.zip", + listOfPaths = List( + wd / "File.txt", + wd / "folder1" + ) + ) +// +// // Adding files and folders to an existing zip file +// val zipFile2: os.Path = os.zipIn( +// destination = wd / "zip-file1-test.zip", +// listOfPaths = List( +// wd / "folder2", +// wd / "Multi Line.txt" +// ) +// ) + + // Unzip +// print("Test Done") + } + } + } + +} From 4eb1c5fb2f50b8e5e1c83c3ea3fcd774a4398a3f Mon Sep 17 00:00:00 2001 From: Chaitanya Waikar Date: Sun, 29 Sep 2024 13:35:31 +0200 Subject: [PATCH 02/15] Extend the `zipIn` functionality to append new files and folders to an existing zip file --- os/src/ZipOps.scala | 65 +++++++++++++++++++++++++++++------- os/test/src/ZipOpTests.scala | 23 +++++++------ 2 files changed, 65 insertions(+), 23 deletions(-) diff --git a/os/src/ZipOps.scala b/os/src/ZipOps.scala index 332394db..1f2cdcdd 100644 --- a/os/src/ZipOps.scala +++ b/os/src/ZipOps.scala @@ -1,9 +1,8 @@ package os -import java.io.{FileInputStream, FileOutputStream} -import java.net.URI +import java.io.{FileInputStream, FileOutputStream, ByteArrayInputStream, ByteArrayOutputStream} import java.nio.file.{Files, Path, StandardCopyOption} -import java.util.zip.{ZipEntry, ZipOutputStream} +import java.util.zip.{ZipEntry, ZipFile, ZipOutputStream} object zipIn { @@ -17,25 +16,71 @@ object zipIn { val zipFilePath: java.nio.file.Path = resolveDestinationZipFile(javaNIODestination) - // Create a zip output stream - val zipOut = new ZipOutputStream(new FileOutputStream(zipFilePath.toFile)) + // Determine if we need to append to the existing zip file + if (options.contains("-u") && Files.exists(zipFilePath)) { + // Append mode: Read existing entries, then add new entries + appendToExistingZip(zipFilePath, pathsToBeZipped) + } else { + // Create a new zip file + createNewZip(zipFilePath, pathsToBeZipped) + } + + os.Path(zipFilePath) // Return the path to the created or updated zip file + } + private def createNewZip(zipFilePath: java.nio.file.Path, pathsToBeZipped: List[java.nio.file.Path]): Unit = { + val zipOut = new ZipOutputStream(new FileOutputStream(zipFilePath.toFile)) try { pathsToBeZipped.foreach { path => val file = path.toFile if (file.isDirectory) { - // Zip the folder recursively zipFolder(file, file.getName, zipOut) } else { - // Zip the individual file zipFile(file, zipOut) } } } finally { zipOut.close() } + } + + private def appendToExistingZip(zipFilePath: java.nio.file.Path, pathsToBeZipped: List[java.nio.file.Path]): Unit = { + // Temporary storage for the original zip entries + val tempOut = new ByteArrayOutputStream() + val zipOut = new ZipOutputStream(tempOut) + + val existingZip = new ZipFile(zipFilePath.toFile) + + // Copy existing entries + existingZip.entries().asIterator().forEachRemaining { entry => + val inputStream = existingZip.getInputStream(entry) + zipOut.putNextEntry(new ZipEntry(entry.getName)) + + val buffer = new Array[Byte](1024) + var length = inputStream.read(buffer) + while (length > 0) { + zipOut.write(buffer, 0, length) + length = inputStream.read(buffer) + } + inputStream.close() + zipOut.closeEntry() + } + + // Append new files and folders + pathsToBeZipped.foreach { path => + val file = path.toFile + if (file.isDirectory) { + zipFolder(file, file.getName, zipOut) + } else { + zipFile(file, zipOut) + } + } + + zipOut.close() - os.Path(zipFilePath) // Return the path to the created zip file + // Write the updated zip content back to the original zip file + val newZipContent = tempOut.toByteArray + Files.write(zipFilePath, newZipContent) } private def zipFile(file: java.io.File, zipOut: ZipOutputStream): Unit = { @@ -43,7 +88,6 @@ object zipIn { val zipEntry = new ZipEntry(file.getName) zipOut.putNextEntry(zipEntry) - // Define a 1KB buffer for reading file content val buffer = new Array[Byte](1024) var length = fis.read(buffer) while (length >= 0) { @@ -59,15 +103,12 @@ object zipIn { if (files != null) { files.foreach { file => if (file.isDirectory) { - // Recursively zip subdirectory zipFolder(file, parentFolderName + "/" + file.getName, zipOut) } else { - // Add the file with its relative path val fis = new FileInputStream(file) val zipEntry = new ZipEntry(parentFolderName + "/" + file.getName) zipOut.putNextEntry(zipEntry) - // Write file content to zip val buffer = new Array[Byte](1024) var length = fis.read(buffer) while (length >= 0) { diff --git a/os/test/src/ZipOpTests.scala b/os/test/src/ZipOpTests.scala index c8a7a56c..d22ebb8f 100644 --- a/os/test/src/ZipOpTests.scala +++ b/os/test/src/ZipOpTests.scala @@ -14,24 +14,25 @@ object ZipOpTests extends TestSuite { test - prep { wd => // Zipping files and folders in a new zip file val zipFile1: os.Path = os.zipIn( - destination = wd / "zip-file1-test.zip", + destination = wd / "zip-file-test.zip", listOfPaths = List( wd / "File.txt", wd / "folder1" ) ) -// -// // Adding files and folders to an existing zip file -// val zipFile2: os.Path = os.zipIn( -// destination = wd / "zip-file1-test.zip", -// listOfPaths = List( -// wd / "folder2", -// wd / "Multi Line.txt" -// ) -// ) + + // Adding files and folders to an existing zip file + val zipFile2: os.Path = os.zipIn( + destination = wd / "zip-file-test.zip", + listOfPaths = List( + wd / "folder2", + wd / "Multi Line.txt" + ), + options = List("-u") + ) // Unzip -// print("Test Done") + print("Test Done") } } } From db8db8abeab0b75e899a64ead6599d342196a34b Mon Sep 17 00:00:00 2001 From: Chaitanya Waikar Date: Sun, 29 Sep 2024 15:52:38 +0200 Subject: [PATCH 03/15] Implement the `unzip` functionality. This method would create a new directory if the target directory is not present. --- os/src/ZipOps.scala | 96 +++++++++++++++++++++++++++++++++++- os/test/src/ZipOpTests.scala | 19 +++++-- 2 files changed, 109 insertions(+), 6 deletions(-) diff --git a/os/src/ZipOps.scala b/os/src/ZipOps.scala index 1f2cdcdd..09a807e4 100644 --- a/os/src/ZipOps.scala +++ b/os/src/ZipOps.scala @@ -1,8 +1,8 @@ package os -import java.io.{FileInputStream, FileOutputStream, ByteArrayInputStream, ByteArrayOutputStream} +import java.io._ import java.nio.file.{Files, Path, StandardCopyOption} -import java.util.zip.{ZipEntry, ZipFile, ZipOutputStream} +import java.util.zip.{ZipEntry, ZipFile, ZipInputStream, ZipOutputStream} object zipIn { @@ -132,3 +132,95 @@ object zipIn { zipFilePath } } + +object unzip { + + def apply(source: os.Path, destination: Option[os.Path]): os.Path = { + val sourcePath: java.nio.file.Path = source.toNIO + + // Ensure the source file is a zip file + validateZipFile(sourcePath) + + // Determine the destination directory + val destPath = destination match { + case Some(dest) => createDestinationDirectory(dest) + case None => sourcePath.getParent // Unzip in the same directory as the source + } + + // Perform the unzip operation + unzipFile(sourcePath, destPath) + os.Path(destPath) + } + + /** Validates if the input file is a valid zip file */ + private def validateZipFile(sourcePath: java.nio.file.Path): Unit = { + if (!Files.exists(sourcePath)) { + throw new IllegalArgumentException(s"Source file does not exist: $sourcePath") + } + if (!sourcePath.toString.endsWith(".zip")) { + throw new IllegalArgumentException(s"Source file is not a zip file: $sourcePath") + } + } + + /** Creates the destination directory if it doesn't exist */ + private def createDestinationDirectory(destination: os.Path): java.nio.file.Path = { + val destPath: java.nio.file.Path = destination.toNIO + if (!Files.exists(destPath)) { + Files.createDirectories(destPath) // Create directory if absent + } + destPath + } + + /** Unzips the file to the destination directory */ + private def unzipFile(sourcePath: java.nio.file.Path, destPath: java.nio.file.Path): Unit = { + val zipInputStream = new ZipInputStream(new FileInputStream(sourcePath.toFile)) + + try { + var zipEntry: ZipEntry = zipInputStream.getNextEntry + while (zipEntry != null) { + val newFile = createFileForEntry(destPath, zipEntry) + if (zipEntry.isDirectory) { + // Create the directory + Files.createDirectories(newFile.toPath) + } else { + // Extract the file + extractFile(zipInputStream, newFile) + } + zipEntry = zipInputStream.getNextEntry + } + } finally { + zipInputStream.closeEntry() + zipInputStream.close() + } + } + + /** Creates the file for the current ZipEntry, preserving the directory structure */ + private def createFileForEntry(destDir: java.nio.file.Path, zipEntry: ZipEntry): File = { + val newFile = new File(destDir.toFile, zipEntry.getName) + val destDirPath = destDir.toFile.getCanonicalPath + val newFilePath = newFile.getCanonicalPath + + // Ensure that the file path is within the destination directory to avoid Zip Slip vulnerability + if (!newFilePath.startsWith(destDirPath)) { + throw new SecurityException(s"Entry is outside of the target directory: ${zipEntry.getName}") + } + newFile + } + + /** Extracts the file content from the zip to the destination */ + private def extractFile(zipInputStream: ZipInputStream, file: File): Unit = { + val parentDir = file.getParentFile + if (!parentDir.exists()) { + parentDir.mkdirs() // Ensure parent directories exist + } + + val outputStream = Files.newOutputStream(file.toPath) + val buffer = new Array[Byte](1024) + var len = zipInputStream.read(buffer) + while (len > 0) { + outputStream.write(buffer, 0, len) + len = zipInputStream.read(buffer) + } + outputStream.close() + } +} diff --git a/os/test/src/ZipOpTests.scala b/os/test/src/ZipOpTests.scala index d22ebb8f..7a49ae48 100644 --- a/os/test/src/ZipOpTests.scala +++ b/os/test/src/ZipOpTests.scala @@ -13,8 +13,9 @@ object ZipOpTests extends TestSuite { test - prep { wd => // Zipping files and folders in a new zip file + val zipFileName = "zip-file-test.zip" val zipFile1: os.Path = os.zipIn( - destination = wd / "zip-file-test.zip", + destination = wd / zipFileName, listOfPaths = List( wd / "File.txt", wd / "folder1" @@ -23,7 +24,7 @@ object ZipOpTests extends TestSuite { // Adding files and folders to an existing zip file val zipFile2: os.Path = os.zipIn( - destination = wd / "zip-file-test.zip", + destination = wd / zipFileName, listOfPaths = List( wd / "folder2", wd / "Multi Line.txt" @@ -31,8 +32,18 @@ object ZipOpTests extends TestSuite { options = List("-u") ) - // Unzip - print("Test Done") + // Unzip file to a destination folder + val unzippedFolder = os.unzip( + source = wd / zipFileName, + destination = Some(wd / "unzipped folder") + ) + + val paths = os.walk(unzippedFolder) + assert(paths.length == 9) + assert(paths.contains(unzippedFolder / "File.txt")) + assert(paths.contains(unzippedFolder / "Multi Line.txt")) + assert(paths.contains(unzippedFolder / "folder1" / "one.txt")) + assert(paths.contains(unzippedFolder / "folder2" / "nestedB" / "b.txt")) } } } From 997444af0e71811b374aef3a143fd010f7c4219c Mon Sep 17 00:00:00 2001 From: Chaitanya Waikar Date: Sun, 29 Sep 2024 16:35:31 +0200 Subject: [PATCH 04/15] Refactoring zip functionality. - Rename `zipIn` to `zip` - Remove `options` parameter and replace with individual flags i.e `appendToExisting` --- os/src/ZipOps.scala | 6 +++--- os/test/src/ZipOpTests.scala | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/os/src/ZipOps.scala b/os/src/ZipOps.scala index 09a807e4..e9a84d84 100644 --- a/os/src/ZipOps.scala +++ b/os/src/ZipOps.scala @@ -4,12 +4,12 @@ import java.io._ import java.nio.file.{Files, Path, StandardCopyOption} import java.util.zip.{ZipEntry, ZipFile, ZipInputStream, ZipOutputStream} -object zipIn { +object zip { def apply( destination: os.Path, listOfPaths: List[os.Path], - options: List[String] = List.empty + appendToExisting: Boolean = false ): os.Path = { val javaNIODestination: java.nio.file.Path = destination.toNIO val pathsToBeZipped: List[java.nio.file.Path] = listOfPaths.map(_.toNIO) @@ -17,7 +17,7 @@ object zipIn { val zipFilePath: java.nio.file.Path = resolveDestinationZipFile(javaNIODestination) // Determine if we need to append to the existing zip file - if (options.contains("-u") && Files.exists(zipFilePath)) { + if (appendToExisting && Files.exists(zipFilePath)) { // Append mode: Read existing entries, then add new entries appendToExistingZip(zipFilePath, pathsToBeZipped) } else { diff --git a/os/test/src/ZipOpTests.scala b/os/test/src/ZipOpTests.scala index 7a49ae48..72361c4f 100644 --- a/os/test/src/ZipOpTests.scala +++ b/os/test/src/ZipOpTests.scala @@ -14,7 +14,7 @@ object ZipOpTests extends TestSuite { test - prep { wd => // Zipping files and folders in a new zip file val zipFileName = "zip-file-test.zip" - val zipFile1: os.Path = os.zipIn( + val zipFile1: os.Path = os.zip( destination = wd / zipFileName, listOfPaths = List( wd / "File.txt", @@ -23,13 +23,13 @@ object ZipOpTests extends TestSuite { ) // Adding files and folders to an existing zip file - val zipFile2: os.Path = os.zipIn( + val zipFile2: os.Path = os.zip( destination = wd / zipFileName, listOfPaths = List( wd / "folder2", wd / "Multi Line.txt" ), - options = List("-u") + appendToExisting = true ) // Unzip file to a destination folder From ec795f33877790eeebc34c0d7ebf9a5c6009090e Mon Sep 17 00:00:00 2001 From: Chaitanya Waikar Date: Sun, 29 Sep 2024 22:21:59 +0200 Subject: [PATCH 05/15] Introduce new options in zip functionality - Include only given files during zipping (similar to -i option in zip) - Exclude given files during zipping (similar to -x option in zip) - Delete given files from the zip archieve (similar to -d option in zip) Tests added along with the implementation --- os/src/ZipOps.scala | 176 +++++++++++++++++++++++++---------- os/test/src/ZipOpTests.scala | 88 +++++++++++++++++- 2 files changed, 212 insertions(+), 52 deletions(-) diff --git a/os/src/ZipOps.scala b/os/src/ZipOps.scala index e9a84d84..38707dc0 100644 --- a/os/src/ZipOps.scala +++ b/os/src/ZipOps.scala @@ -3,40 +3,58 @@ package os import java.io._ import java.nio.file.{Files, Path, StandardCopyOption} import java.util.zip.{ZipEntry, ZipFile, ZipInputStream, ZipOutputStream} +import scala.util.matching.Regex object zip { def apply( destination: os.Path, - listOfPaths: List[os.Path], - appendToExisting: Boolean = false + listOfPaths: List[os.Path] = List(), + appendToExisting: Boolean = false, + excludePatterns: List[String] = List(), // -x option + includePatterns: List[String] = List(), // -i option + deletePatterns: List[String] = List() // -d option ): os.Path = { + val javaNIODestination: java.nio.file.Path = destination.toNIO val pathsToBeZipped: List[java.nio.file.Path] = listOfPaths.map(_.toNIO) + // Convert the string patterns into regex + val excludeRegexPatterns: List[Regex] = excludePatterns.map(_.r) + val includeRegexPatterns: List[Regex] = includePatterns.map(_.r) + val deleteRegexPatterns: List[Regex] = deletePatterns.map(_.r) + val zipFilePath: java.nio.file.Path = resolveDestinationZipFile(javaNIODestination) - // Determine if we need to append to the existing zip file + if (Files.exists(zipFilePath) && deletePatterns.nonEmpty) { + deleteFilesFromZip(zipFilePath, deleteRegexPatterns) + } + if (appendToExisting && Files.exists(zipFilePath)) { - // Append mode: Read existing entries, then add new entries - appendToExistingZip(zipFilePath, pathsToBeZipped) + appendToExistingZip(zipFilePath, pathsToBeZipped, excludeRegexPatterns, includeRegexPatterns) } else { - // Create a new zip file - createNewZip(zipFilePath, pathsToBeZipped) + createNewZip(zipFilePath, pathsToBeZipped, excludeRegexPatterns, includeRegexPatterns) } - os.Path(zipFilePath) // Return the path to the created or updated zip file + os.Path(zipFilePath) } - private def createNewZip(zipFilePath: java.nio.file.Path, pathsToBeZipped: List[java.nio.file.Path]): Unit = { + private def createNewZip( + zipFilePath: java.nio.file.Path, + pathsToBeZipped: List[java.nio.file.Path], + excludePatterns: List[Regex], + includePatterns: List[Regex] + ): Unit = { val zipOut = new ZipOutputStream(new FileOutputStream(zipFilePath.toFile)) try { pathsToBeZipped.foreach { path => val file = path.toFile - if (file.isDirectory) { - zipFolder(file, file.getName, zipOut) - } else { - zipFile(file, zipOut) + if (shouldInclude(file.getName, excludePatterns, includePatterns)) { + if (file.isDirectory) { + zipFolder(file, file.getName, zipOut, excludePatterns, includePatterns) + } else { + zipFile(file, zipOut) + } } } } finally { @@ -44,45 +62,91 @@ object zip { } } - private def appendToExistingZip(zipFilePath: java.nio.file.Path, pathsToBeZipped: List[java.nio.file.Path]): Unit = { - // Temporary storage for the original zip entries + private def appendToExistingZip( + zipFilePath: java.nio.file.Path, + pathsToBeZipped: List[java.nio.file.Path], + excludePatterns: List[Regex], + includePatterns: List[Regex] + ): Unit = { val tempOut = new ByteArrayOutputStream() val zipOut = new ZipOutputStream(tempOut) val existingZip = new ZipFile(zipFilePath.toFile) - // Copy existing entries existingZip.entries().asIterator().forEachRemaining { entry => - val inputStream = existingZip.getInputStream(entry) - zipOut.putNextEntry(new ZipEntry(entry.getName)) - - val buffer = new Array[Byte](1024) - var length = inputStream.read(buffer) - while (length > 0) { - zipOut.write(buffer, 0, length) - length = inputStream.read(buffer) + if (shouldInclude(entry.getName, excludePatterns, includePatterns)) { + val inputStream = existingZip.getInputStream(entry) + zipOut.putNextEntry(new ZipEntry(entry.getName)) + + val buffer = new Array[Byte](1024) + var length = inputStream.read(buffer) + while (length > 0) { + zipOut.write(buffer, 0, length) + length = inputStream.read(buffer) + } + inputStream.close() + zipOut.closeEntry() } - inputStream.close() - zipOut.closeEntry() } - // Append new files and folders pathsToBeZipped.foreach { path => val file = path.toFile - if (file.isDirectory) { - zipFolder(file, file.getName, zipOut) - } else { - zipFile(file, zipOut) + if (shouldInclude(file.getName, excludePatterns, includePatterns)) { + if (file.isDirectory) { + zipFolder(file, file.getName, zipOut, excludePatterns, includePatterns) + } else { + zipFile(file, zipOut) + } + } + } + + zipOut.close() + + val newZipContent = tempOut.toByteArray + Files.write(zipFilePath, newZipContent) + } + + private def deleteFilesFromZip( + zipFilePath: java.nio.file.Path, + deletePatterns: List[Regex] + ): Unit = { + val tempOut = new ByteArrayOutputStream() + val zipOut = new ZipOutputStream(tempOut) + + val existingZip = new ZipFile(zipFilePath.toFile) + + existingZip.entries().asIterator().forEachRemaining { entry => + if (!deletePatterns.exists(_.findFirstIn(entry.getName).isDefined)) { + val inputStream = existingZip.getInputStream(entry) + zipOut.putNextEntry(new ZipEntry(entry.getName)) + + val buffer = new Array[Byte](1024) + var length = inputStream.read(buffer) + while (length > 0) { + zipOut.write(buffer, 0, length) + length = inputStream.read(buffer) + } + inputStream.close() + zipOut.closeEntry() } } zipOut.close() - // Write the updated zip content back to the original zip file val newZipContent = tempOut.toByteArray Files.write(zipFilePath, newZipContent) } + private def shouldInclude( + fileName: String, + excludePatterns: List[Regex], + includePatterns: List[Regex] + ): Boolean = { + val isExcluded = excludePatterns.exists(_.findFirstIn(fileName).isDefined) + val isIncluded = includePatterns.isEmpty || includePatterns.exists(_.findFirstIn(fileName).isDefined) + !isExcluded && isIncluded + } + private def zipFile(file: java.io.File, zipOut: ZipOutputStream): Unit = { val fis = new FileInputStream(file) val zipEntry = new ZipEntry(file.getName) @@ -98,36 +162,43 @@ object zip { fis.close() } - private def zipFolder(folder: java.io.File, parentFolderName: String, zipOut: ZipOutputStream): Unit = { + private def zipFolder( + folder: java.io.File, + parentFolderName: String, + zipOut: ZipOutputStream, + excludePatterns: List[Regex], + includePatterns: List[Regex] + ): Unit = { val files = folder.listFiles() if (files != null) { files.foreach { file => - if (file.isDirectory) { - zipFolder(file, parentFolderName + "/" + file.getName, zipOut) - } else { - val fis = new FileInputStream(file) - val zipEntry = new ZipEntry(parentFolderName + "/" + file.getName) - zipOut.putNextEntry(zipEntry) - - val buffer = new Array[Byte](1024) - var length = fis.read(buffer) - while (length >= 0) { - zipOut.write(buffer, 0, length) - length = fis.read(buffer) + if (shouldInclude(file.getName, excludePatterns, includePatterns)) { + if (file.isDirectory) { + zipFolder(file, parentFolderName + "/" + file.getName, zipOut, excludePatterns, includePatterns) + } else { + val fis = new FileInputStream(file) + val zipEntry = new ZipEntry(parentFolderName + "/" + file.getName) + zipOut.putNextEntry(zipEntry) + + val buffer = new Array[Byte](1024) + var length = fis.read(buffer) + while (length >= 0) { + zipOut.write(buffer, 0, length) + length = fis.read(buffer) + } + + fis.close() } - - fis.close() } } } } private def resolveDestinationZipFile(destination: java.nio.file.Path): java.nio.file.Path = { - // Check if destination is a directory or file val zipFilePath: java.nio.file.Path = if (Files.isDirectory(destination)) { - destination.resolve("archive.zip") // Append default zip name + destination.resolve("archive.zip") } else { - destination // Use provided file name + destination } zipFilePath } @@ -135,7 +206,10 @@ object zip { object unzip { - def apply(source: os.Path, destination: Option[os.Path]): os.Path = { + def apply( + source: os.Path, + destination: Option[os.Path] = None + ): os.Path = { val sourcePath: java.nio.file.Path = source.toNIO // Ensure the source file is a zip file diff --git a/os/test/src/ZipOpTests.scala b/os/test/src/ZipOpTests.scala index 72361c4f..6afb5785 100644 --- a/os/test/src/ZipOpTests.scala +++ b/os/test/src/ZipOpTests.scala @@ -9,7 +9,7 @@ object ZipOpTests extends TestSuite { def tests = Tests { - test("zipOps") { + test("zipAndUnzipFolder") { test - prep { wd => // Zipping files and folders in a new zip file @@ -46,6 +46,92 @@ object ZipOpTests extends TestSuite { assert(paths.contains(unzippedFolder / "folder2" / "nestedB" / "b.txt")) } } + + test("zipByExcludingCertainFiles") { + + test - prep { wd => + + val amxFile = "File.amx" + os.copy(wd / "File.txt", wd / amxFile) + + // Zipping files and folders in a new zip file + val zipFileName = "zipByExcludingCertainFiles.zip" + val zipFile1: os.Path = os.zip( + destination = wd / zipFileName, + listOfPaths = List( + wd / "File.txt", + wd / amxFile, + wd / "Multi Line.txt" + ), + excludePatterns = List(".*\\.txt") + ) + + // Unzip file to check for contents + val outputZipFilePath = os.unzip(zipFile1, destination = Some(wd / "zipByExcludingCertainFiles")) + val paths = os.walk(outputZipFilePath) + assert(paths.length == 1) + assert(paths.contains(outputZipFilePath / amxFile)) + } + } + + test("zipByIncludingCertainFiles") { + + test - prep { wd => + + val amxFile = "File.amx" + os.copy(wd / "File.txt", wd / amxFile) + + // Zipping files and folders in a new zip file + val zipFileName = "zipByIncludingCertainFiles.zip" + val zipFile1: os.Path = os.zip( + destination = wd / zipFileName, + listOfPaths = List( + wd / "File.txt", + wd / amxFile, + wd / "Multi Line.txt" + ), + includePatterns = List(".*\\.amx") + ) + + // Unzip file to check for contents + val outputZipFilePath = os.unzip(zipFile1, destination = Some(wd / "zipByIncludingCertainFiles")) + val paths = os.walk(outputZipFilePath) + assert(paths.length == 1) + assert(paths.contains(outputZipFilePath / amxFile)) + } + } + +// test("zipByDeletingCertainFiles") { +// +// test - prep { wd => +// +// val amxFile = "File.amx" +// os.copy(wd / "File.txt", wd / amxFile) +// +// // Zipping files and folders in a new zip file +// val zipFileName = "zipByDeletingCertainFiles.zip" +// val zipFile1: os.Path = os.zip( +// destination = wd / zipFileName, +// listOfPaths = List( +// wd / "File.txt", +// wd / amxFile, +// wd / "Multi Line.txt" +// ) +// ) +// +// os.zip( +// destination = zipFile1, +// deletePatterns = List(amxFile) +// ) +// +// // Unzip file to check for contents +// val outputZipFilePath = os.unzip(zipFile1, destination = Some(wd / "zipByDeletingCertainFiles")) +// val paths = os.walk(outputZipFilePath) +// assert(paths.length == 2) +// assert(paths.contains(outputZipFilePath / "File.txt")) +// assert(paths.contains(outputZipFilePath / "Multi Line.txt")) +// } +// } } } From 24d6ece39aba4473853ac0df2781d8a82ba71255 Mon Sep 17 00:00:00 2001 From: Chaitanya Waikar Date: Sun, 29 Sep 2024 22:42:33 +0200 Subject: [PATCH 06/15] Fix delete option in zip files --- os/src/ZipOps.scala | 12 +++---- os/test/src/ZipOpTests.scala | 62 ++++++++++++++++++------------------ 2 files changed, 37 insertions(+), 37 deletions(-) diff --git a/os/src/ZipOps.scala b/os/src/ZipOps.scala index 38707dc0..1d08d30d 100644 --- a/os/src/ZipOps.scala +++ b/os/src/ZipOps.scala @@ -28,14 +28,14 @@ object zip { if (Files.exists(zipFilePath) && deletePatterns.nonEmpty) { deleteFilesFromZip(zipFilePath, deleteRegexPatterns) - } - - if (appendToExisting && Files.exists(zipFilePath)) { - appendToExistingZip(zipFilePath, pathsToBeZipped, excludeRegexPatterns, includeRegexPatterns) } else { - createNewZip(zipFilePath, pathsToBeZipped, excludeRegexPatterns, includeRegexPatterns) - } + if (appendToExisting && Files.exists(zipFilePath)) { + appendToExistingZip(zipFilePath, pathsToBeZipped, excludeRegexPatterns, includeRegexPatterns) + } else { + createNewZip(zipFilePath, pathsToBeZipped, excludeRegexPatterns, includeRegexPatterns) + } + } os.Path(zipFilePath) } diff --git a/os/test/src/ZipOpTests.scala b/os/test/src/ZipOpTests.scala index 6afb5785..60cd14eb 100644 --- a/os/test/src/ZipOpTests.scala +++ b/os/test/src/ZipOpTests.scala @@ -101,37 +101,37 @@ object ZipOpTests extends TestSuite { } } -// test("zipByDeletingCertainFiles") { -// -// test - prep { wd => -// -// val amxFile = "File.amx" -// os.copy(wd / "File.txt", wd / amxFile) -// -// // Zipping files and folders in a new zip file -// val zipFileName = "zipByDeletingCertainFiles.zip" -// val zipFile1: os.Path = os.zip( -// destination = wd / zipFileName, -// listOfPaths = List( -// wd / "File.txt", -// wd / amxFile, -// wd / "Multi Line.txt" -// ) -// ) -// -// os.zip( -// destination = zipFile1, -// deletePatterns = List(amxFile) -// ) -// -// // Unzip file to check for contents -// val outputZipFilePath = os.unzip(zipFile1, destination = Some(wd / "zipByDeletingCertainFiles")) -// val paths = os.walk(outputZipFilePath) -// assert(paths.length == 2) -// assert(paths.contains(outputZipFilePath / "File.txt")) -// assert(paths.contains(outputZipFilePath / "Multi Line.txt")) -// } -// } + test("zipByDeletingCertainFiles") { + + test - prep { wd => + + val amxFile = "File.amx" + os.copy(wd / "File.txt", wd / amxFile) + + // Zipping files and folders in a new zip file + val zipFileName = "zipByDeletingCertainFiles.zip" + val zipFile1: os.Path = os.zip( + destination = wd / zipFileName, + listOfPaths = List( + wd / "File.txt", + wd / amxFile, + wd / "Multi Line.txt" + ) + ) + + os.zip( + destination = zipFile1, + deletePatterns = List(amxFile) + ) + + // Unzip file to check for contents + val outputZipFilePath = os.unzip(zipFile1, destination = Some(wd / "zipByDeletingCertainFiles")) + val paths = os.walk(outputZipFilePath) + assert(paths.length == 2) + assert(paths.contains(outputZipFilePath / "File.txt")) + assert(paths.contains(outputZipFilePath / "Multi Line.txt")) + } + } } } From e76401ad98db5ba093adceee7d68285942262d3d Mon Sep 17 00:00:00 2001 From: Chaitanya Waikar Date: Sun, 29 Sep 2024 23:59:17 +0200 Subject: [PATCH 07/15] Enhance unzip functionality with - listing files without unzipping - excluding certain files/file regex during unzipping --- os/src/ZipOps.scala | 54 ++++++++++++++++++++++++------ os/test/src/ZipOpTests.scala | 64 +++++++++++++++++++++++++++++++++++- 2 files changed, 107 insertions(+), 11 deletions(-) diff --git a/os/src/ZipOps.scala b/os/src/ZipOps.scala index 1d08d30d..6d605c44 100644 --- a/os/src/ZipOps.scala +++ b/os/src/ZipOps.scala @@ -208,13 +208,25 @@ object unzip { def apply( source: os.Path, - destination: Option[os.Path] = None + destination: Option[os.Path] = None, + excludePatterns: List[String] = List(), // Patterns to exclude + listOnly: Boolean = false // List contents without extracting ): os.Path = { + val sourcePath: java.nio.file.Path = source.toNIO // Ensure the source file is a zip file validateZipFile(sourcePath) + // Convert the exclusion patterns to regex + val excludeRegexPatterns: List[Regex] = excludePatterns.map(_.r) + + // If listOnly is true, list the contents and return the source path + if (listOnly) { + listContents(sourcePath, excludeRegexPatterns) + return source + } + // Determine the destination directory val destPath = destination match { case Some(dest) => createDestinationDirectory(dest) @@ -222,7 +234,7 @@ object unzip { } // Perform the unzip operation - unzipFile(sourcePath, destPath) + unzipFile(sourcePath, destPath, excludeRegexPatterns) os.Path(destPath) } @@ -246,19 +258,25 @@ object unzip { } /** Unzips the file to the destination directory */ - private def unzipFile(sourcePath: java.nio.file.Path, destPath: java.nio.file.Path): Unit = { + private def unzipFile( + sourcePath: java.nio.file.Path, + destPath: java.nio.file.Path, + excludePatterns: List[Regex] + ): Unit = { + val zipInputStream = new ZipInputStream(new FileInputStream(sourcePath.toFile)) try { var zipEntry: ZipEntry = zipInputStream.getNextEntry while (zipEntry != null) { - val newFile = createFileForEntry(destPath, zipEntry) - if (zipEntry.isDirectory) { - // Create the directory - Files.createDirectories(newFile.toPath) - } else { - // Extract the file - extractFile(zipInputStream, newFile) + // Skip files that match the exclusion patterns + if (!shouldExclude(zipEntry.getName, excludePatterns)) { + val newFile = createFileForEntry(destPath, zipEntry) + if (zipEntry.isDirectory) { + Files.createDirectories(newFile.toPath) + } else { + extractFile(zipInputStream, newFile) + } } zipEntry = zipInputStream.getNextEntry } @@ -268,6 +286,17 @@ object unzip { } } + /** Lists the contents of the zip file */ + private def listContents(sourcePath: java.nio.file.Path, excludePatterns: List[Regex]): Unit = { + val zipFile = new ZipFile(sourcePath.toFile) + + zipFile.entries().asIterator().forEachRemaining { entry => + if (!shouldExclude(entry.getName, excludePatterns)) { + println(entry.getName) + } + } + } + /** Creates the file for the current ZipEntry, preserving the directory structure */ private def createFileForEntry(destDir: java.nio.file.Path, zipEntry: ZipEntry): File = { val newFile = new File(destDir.toFile, zipEntry.getName) @@ -297,4 +326,9 @@ object unzip { } outputStream.close() } + + /** Determines if a file should be excluded based on the given patterns */ + private def shouldExclude(fileName: String, excludePatterns: List[Regex]): Boolean = { + excludePatterns.exists(_.findFirstIn(fileName).isDefined) + } } diff --git a/os/test/src/ZipOpTests.scala b/os/test/src/ZipOpTests.scala index 60cd14eb..17cc650b 100644 --- a/os/test/src/ZipOpTests.scala +++ b/os/test/src/ZipOpTests.scala @@ -3,6 +3,7 @@ package test.os import test.os.TestUtil.prep import utest._ +import java.io.{ByteArrayOutputStream, PrintStream, PrintWriter, StringWriter} import scala.collection.immutable.Seq object ZipOpTests extends TestSuite { @@ -132,6 +133,67 @@ object ZipOpTests extends TestSuite { assert(paths.contains(outputZipFilePath / "Multi Line.txt")) } } - } + test("listContentsOfZipFileWithoutExtracting") { + + test - prep { wd => + // Zipping files and folders in a new zip file + val zipFileName = "listContentsOfZipFileWithoutExtracting.zip" + val zipFile: os.Path = os.zip( + destination = wd / zipFileName, + listOfPaths = List( + wd / "File.txt", + wd / "folder1" + ) + ) + val originalOut = System.out + val outputStream = new ByteArrayOutputStream() + System.setOut(new PrintStream(outputStream)) + + // Unzip file to a destination folder + val unzippedFolder = os.unzip( + source = wd / zipFileName, + listOnly = true + ) + + // Then + val capturedOutput: Array[String] = outputStream.toString.split("\n") + assert(capturedOutput(0) == "File.txt") + assert(capturedOutput(1) == "folder1/one.txt") + + // Restore the original output stream + System.setOut(originalOut) + } + } + + test("unzipAllExceptExcludingCertainFiles") { + + test - prep { wd => + val amxFile = "File.amx" + os.copy(wd / "File.txt", wd / amxFile) + + val zipFileName = "unzipAllExceptExcludingCertainFiles.zip" + val zipFile: os.Path = os.zip( + destination = wd / zipFileName, + listOfPaths = List( + wd / "File.txt", + wd / amxFile, + wd / "folder1" + ) + ) + + // Unzip file to a destination folder + val unzippedFolder = os.unzip( + source = wd / zipFileName, + destination = Some(wd / "unzipAllExceptExcludingCertainFiles"), + excludePatterns = List(amxFile) + ) + + val paths = os.walk(unzippedFolder) + assert(paths.length == 3) + assert(paths.contains(unzippedFolder / "File.txt")) + assert(paths.contains(unzippedFolder / "folder1" / "one.txt")) + } + } + } } From 8f0cd994bbdd7e9528da9234c11c16ad29cedc7b Mon Sep 17 00:00:00 2001 From: Chaitanya Waikar Date: Mon, 30 Sep 2024 00:23:41 +0200 Subject: [PATCH 08/15] Add documentation support for zip and unzip functionality. --- Readme.adoc | 146 +++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 122 insertions(+), 24 deletions(-) diff --git a/Readme.adoc b/Readme.adoc index 103afbea..17628e14 100644 --- a/Readme.adoc +++ b/Readme.adoc @@ -985,9 +985,9 @@ os.remove(target: Path, checkExists: Boolean = false): Boolean ---- Remove the target file or folder. Folders need to be empty to be removed; if you -want to remove a folder tree recursively, use <>. +want to remove a folder tree recursively, use <>. Returns `true` if the file was present before. -It will fail with an exception when the file is missing but `checkExists` is `true`, +It will fail with an exception when the file is missing but `checkExists` is `true`, or when the directory to remove is not empty. [source,scala] @@ -1215,6 +1215,104 @@ os.write(tempDir / "file", "Hello") os.list(tempDir) ==> Seq(tempDir / "file") ---- +=== Zip & Unzip Files + +==== `os.zip` + +[source,scala] +---- +os.zip(destination: os.Path, + listOfPaths: List[os.Path] = List(), + appendToExisting: Boolean = false, + excludePatterns: List[String] = List(), + includePatterns: List[String] = List(), + deletePatterns: List[String] = List()): Path +---- + +The zip object provides functionality to create or modify zip archives. It supports: + +- Zipping Files and Directories: You can zip both individual files and entire directories. +- Appending to Existing Archives: Files can be appended to an existing zip archive. +- Exclude Patterns (-x): You can specify files or patterns to exclude while zipping. +- Include Patterns (-i): You can include specific files or patterns while zipping. +- Delete Patterns (-d): You can delete specific files from an existing zip archive. + +===== Zipping Files and Folders +[source,scala] +---- +val pathsToZip = List( + os.Path("/path/to/file1.txt"), + os.Path("/path/to/folder") +) +os.zip(os.Path("/path/to/destination.zip"), pathsToZip) +---- + +This will create a new zip archive at `/path/to/destination.zip` containing `file1.txt` and everything inside folder. + +===== Appending to an Existing Zip File +[source,scala] +---- +os.zip( + os.Path("/path/to/existing.zip"), + List(os.Path("/path/to/newfile.txt")), + appendToExisting = true +) +---- + +This will append `newfile.txt` to the existing zip file existing.zip. + +===== Excluding/Including Files in Zip + +You can specify files or folders to be excluded or included when creating the zip: + + +[source,scala] +---- +os.zip( + os.Path("/path/to/destination.zip"), + List(os.Path("/path/to/folder")), + excludePatterns = List(".*\\.log", "temp/.*"), // Exclude log files and "temp" folder + includePatterns = List(".*\\.txt") // Include only .txt files +) + +---- + +This will include only `.txt` files, excluding any `.log` files and anything inside the `temp` folder. + + +==== `os.unzip` + +===== Unzipping Files +[source,scala] +---- +os.unzip(os.Path("/path/to/archive.zip"), Some(os.Path("/path/to/destination"))) +---- + +This extracts the contents of `archive.zip` to the specified destination. + + +===== Excluding Files While Unzipping +You can exclude certain files from being extracted using patterns: + +[source,scala] +---- +os.unzip( + os.Path("/path/to/archive.zip"), + Some(os.Path("/path/to/destination")), + excludePatterns = List(".*\\.log", "temp/.*") // Exclude log files and the "temp" folder +) +---- + +===== Listing Archive Contents +You can list the contents of the zip file without extracting them: + +[source,scala] +---- +os.unzip(os.Path("/path/to/archive.zip"), listOnly = true) +---- + +This will print all the file paths contained in the zip archive. + === Filesystem Metadata ==== `os.stat` @@ -1708,13 +1806,13 @@ val yes10 = os.proc("yes") ---- This feature is implemented inside the library and will terminate any process reading the -stdin of other process in pipeline on every IO error. This behavior can be disabled via the -`handleBrokenPipe` flag on `call` and `spawn` methods. Note that Windows does not support -broken pipe behaviour, so a command like`yes` would run forever. `handleBrokenPipe` is set +stdin of other process in pipeline on every IO error. This behavior can be disabled via the +`handleBrokenPipe` flag on `call` and `spawn` methods. Note that Windows does not support +broken pipe behaviour, so a command like`yes` would run forever. `handleBrokenPipe` is set to false by default on Windows. Both `call` and `spawn` correspond in their behavior to their counterparts in the `os.proc`, -but `spawn` returns the `os.ProcessPipeline` instance instead. It offers the same +but `spawn` returns the `os.ProcessPipeline` instance instead. It offers the same `API` as `SubProcess`, but will operate on the set of processes instead of a single one. `Pipefail` is enabled by default, so if any of the processes in the pipeline fails, the whole @@ -2105,8 +2203,8 @@ explicitly choose to convert relative paths to absolute using some base. ==== Roots and filesystems -If you are using a system that supports different roots of paths, e.g. Windows, -you can use the argument of `os.root` to specify which root you want to use. +If you are using a system that supports different roots of paths, e.g. Windows, +you can use the argument of `os.root` to specify which root you want to use. If not specified, the default root will be used (usually, C on Windows, / on Unix). [source,scala] @@ -2128,11 +2226,11 @@ val fs = FileSystems.newFileSystem(uri, env); val path = os.root("/", fs) / "dir" ---- -Note that the jar file system operations suchs as writing to a file are supported -only on JVM 11+. Depending on the filesystem, some operations may not be supported - -for example, running an `os.proc` with pwd in a jar file won't work. You may also -meet limitations imposed by the implementations - in jar file system, the files are -created only after the file system is closed. Until that, the ones created in your +Note that the jar file system operations suchs as writing to a file are supported +only on JVM 11+. Depending on the filesystem, some operations may not be supported - +for example, running an `os.proc` with pwd in a jar file won't work. You may also +meet limitations imposed by the implementations - in jar file system, the files are +created only after the file system is closed. Until that, the ones created in your program are kept in memory. ==== `os.ResourcePath` @@ -2199,9 +2297,9 @@ By default, the following types of values can be used where-ever ``os.Source``s are required: * Any `geny.Writable` data type: - ** `Array[Byte]` - ** `java.lang.String` (these are treated as UTF-8) - ** `java.io.InputStream` +** `Array[Byte]` +** `java.lang.String` (these are treated as UTF-8) +** `java.io.InputStream` * `java.nio.channels.SeekableByteChannel` * Any `TraversableOnce[T]` of the above: e.g. `Seq[String]`, `List[Array[Byte]]`, etc. @@ -2266,9 +2364,9 @@ string, int or set representations of the `os.PermSet` via: === 0.10.7 * Allow multi-segment paths segments for literals https://github.com/com-lihaoyi/os-lib/pull/297: You - can now write `os.pwd / "foo/bar/qux"` rather than `os.pwd / "foo" / "bar" / "qux"`. Note that this - is only allowed for string literals, and non-literal path segments still need to be wrapped e.g. - `def myString = "foo/bar/qux"; os.pwd / os.SubPath(myString)` for security and safety purposes +can now write `os.pwd / "foo/bar/qux"` rather than `os.pwd / "foo" / "bar" / "qux"`. Note that this +is only allowed for string literals, and non-literal path segments still need to be wrapped e.g. +`def myString = "foo/bar/qux"; os.pwd / os.SubPath(myString)` for security and safety purposes [#0-10-6] === 0.10.6 @@ -2279,23 +2377,23 @@ string, int or set representations of the `os.PermSet` via: === 0.10.5 * Introduce `os.SubProcess.env` `DynamicVariable` to override default `env` - (https://github.com/com-lihaoyi/os-lib/pull/295) +(https://github.com/com-lihaoyi/os-lib/pull/295) [#0-10-4] === 0.10.4 * Add a lightweight syntax for `os.call()` and `os.spawn` APIs - (https://github.com/com-lihaoyi/os-lib/pull/292) +(https://github.com/com-lihaoyi/os-lib/pull/292) * Add a configurable grace period when subprocesses timeout and have to - be terminated to give a chance for shutdown logic to run - (https://github.com/com-lihaoyi/os-lib/pull/286) +be terminated to give a chance for shutdown logic to run +(https://github.com/com-lihaoyi/os-lib/pull/286) [#0-10-3] === 0.10.3 * `os.Inherit` now can be redirected on a threadlocal basis via `os.Inherit.in`, `.out`, or `.err`. - `os.InheritRaw` is available if you do not want the redirects to take effect +`os.InheritRaw` is available if you do not want the redirects to take effect [#0-10-2] From 02ba84b5548974a33420b04eebc9fca1f6825ae0 Mon Sep 17 00:00:00 2001 From: Chaitanya Waikar Date: Mon, 30 Sep 2024 22:48:00 +0200 Subject: [PATCH 09/15] Fix scalafmt file changes --- os/src/ZipOps.scala | 96 ++++++++++++++++++++---------------- os/test/src/ZipOpTests.scala | 12 ++--- 2 files changed, 60 insertions(+), 48 deletions(-) diff --git a/os/src/ZipOps.scala b/os/src/ZipOps.scala index 6d605c44..6fa3f4f3 100644 --- a/os/src/ZipOps.scala +++ b/os/src/ZipOps.scala @@ -8,13 +8,13 @@ import scala.util.matching.Regex object zip { def apply( - destination: os.Path, - listOfPaths: List[os.Path] = List(), - appendToExisting: Boolean = false, - excludePatterns: List[String] = List(), // -x option - includePatterns: List[String] = List(), // -i option - deletePatterns: List[String] = List() // -d option - ): os.Path = { + destination: os.Path, + listOfPaths: List[os.Path] = List(), + appendToExisting: Boolean = false, + excludePatterns: List[String] = List(), // -x option + includePatterns: List[String] = List(), // -i option + deletePatterns: List[String] = List() // -d option + ): os.Path = { val javaNIODestination: java.nio.file.Path = destination.toNIO val pathsToBeZipped: List[java.nio.file.Path] = listOfPaths.map(_.toNIO) @@ -30,7 +30,12 @@ object zip { deleteFilesFromZip(zipFilePath, deleteRegexPatterns) } else { if (appendToExisting && Files.exists(zipFilePath)) { - appendToExistingZip(zipFilePath, pathsToBeZipped, excludeRegexPatterns, includeRegexPatterns) + appendToExistingZip( + zipFilePath, + pathsToBeZipped, + excludeRegexPatterns, + includeRegexPatterns + ) } else { createNewZip(zipFilePath, pathsToBeZipped, excludeRegexPatterns, includeRegexPatterns) } @@ -40,11 +45,11 @@ object zip { } private def createNewZip( - zipFilePath: java.nio.file.Path, - pathsToBeZipped: List[java.nio.file.Path], - excludePatterns: List[Regex], - includePatterns: List[Regex] - ): Unit = { + zipFilePath: java.nio.file.Path, + pathsToBeZipped: List[java.nio.file.Path], + excludePatterns: List[Regex], + includePatterns: List[Regex] + ): Unit = { val zipOut = new ZipOutputStream(new FileOutputStream(zipFilePath.toFile)) try { pathsToBeZipped.foreach { path => @@ -63,11 +68,11 @@ object zip { } private def appendToExistingZip( - zipFilePath: java.nio.file.Path, - pathsToBeZipped: List[java.nio.file.Path], - excludePatterns: List[Regex], - includePatterns: List[Regex] - ): Unit = { + zipFilePath: java.nio.file.Path, + pathsToBeZipped: List[java.nio.file.Path], + excludePatterns: List[Regex], + includePatterns: List[Regex] + ): Unit = { val tempOut = new ByteArrayOutputStream() val zipOut = new ZipOutputStream(tempOut) @@ -107,9 +112,9 @@ object zip { } private def deleteFilesFromZip( - zipFilePath: java.nio.file.Path, - deletePatterns: List[Regex] - ): Unit = { + zipFilePath: java.nio.file.Path, + deletePatterns: List[Regex] + ): Unit = { val tempOut = new ByteArrayOutputStream() val zipOut = new ZipOutputStream(tempOut) @@ -138,12 +143,13 @@ object zip { } private def shouldInclude( - fileName: String, - excludePatterns: List[Regex], - includePatterns: List[Regex] - ): Boolean = { + fileName: String, + excludePatterns: List[Regex], + includePatterns: List[Regex] + ): Boolean = { val isExcluded = excludePatterns.exists(_.findFirstIn(fileName).isDefined) - val isIncluded = includePatterns.isEmpty || includePatterns.exists(_.findFirstIn(fileName).isDefined) + val isIncluded = + includePatterns.isEmpty || includePatterns.exists(_.findFirstIn(fileName).isDefined) !isExcluded && isIncluded } @@ -163,18 +169,24 @@ object zip { } private def zipFolder( - folder: java.io.File, - parentFolderName: String, - zipOut: ZipOutputStream, - excludePatterns: List[Regex], - includePatterns: List[Regex] - ): Unit = { + folder: java.io.File, + parentFolderName: String, + zipOut: ZipOutputStream, + excludePatterns: List[Regex], + includePatterns: List[Regex] + ): Unit = { val files = folder.listFiles() if (files != null) { files.foreach { file => if (shouldInclude(file.getName, excludePatterns, includePatterns)) { if (file.isDirectory) { - zipFolder(file, parentFolderName + "/" + file.getName, zipOut, excludePatterns, includePatterns) + zipFolder( + file, + parentFolderName + "/" + file.getName, + zipOut, + excludePatterns, + includePatterns + ) } else { val fis = new FileInputStream(file) val zipEntry = new ZipEntry(parentFolderName + "/" + file.getName) @@ -207,11 +219,11 @@ object zip { object unzip { def apply( - source: os.Path, - destination: Option[os.Path] = None, - excludePatterns: List[String] = List(), // Patterns to exclude - listOnly: Boolean = false // List contents without extracting - ): os.Path = { + source: os.Path, + destination: Option[os.Path] = None, + excludePatterns: List[String] = List(), // Patterns to exclude + listOnly: Boolean = false // List contents without extracting + ): os.Path = { val sourcePath: java.nio.file.Path = source.toNIO @@ -259,10 +271,10 @@ object unzip { /** Unzips the file to the destination directory */ private def unzipFile( - sourcePath: java.nio.file.Path, - destPath: java.nio.file.Path, - excludePatterns: List[Regex] - ): Unit = { + sourcePath: java.nio.file.Path, + destPath: java.nio.file.Path, + excludePatterns: List[Regex] + ): Unit = { val zipInputStream = new ZipInputStream(new FileInputStream(sourcePath.toFile)) diff --git a/os/test/src/ZipOpTests.scala b/os/test/src/ZipOpTests.scala index 17cc650b..ad38c053 100644 --- a/os/test/src/ZipOpTests.scala +++ b/os/test/src/ZipOpTests.scala @@ -51,7 +51,6 @@ object ZipOpTests extends TestSuite { test("zipByExcludingCertainFiles") { test - prep { wd => - val amxFile = "File.amx" os.copy(wd / "File.txt", wd / amxFile) @@ -68,7 +67,8 @@ object ZipOpTests extends TestSuite { ) // Unzip file to check for contents - val outputZipFilePath = os.unzip(zipFile1, destination = Some(wd / "zipByExcludingCertainFiles")) + val outputZipFilePath = + os.unzip(zipFile1, destination = Some(wd / "zipByExcludingCertainFiles")) val paths = os.walk(outputZipFilePath) assert(paths.length == 1) assert(paths.contains(outputZipFilePath / amxFile)) @@ -78,7 +78,6 @@ object ZipOpTests extends TestSuite { test("zipByIncludingCertainFiles") { test - prep { wd => - val amxFile = "File.amx" os.copy(wd / "File.txt", wd / amxFile) @@ -95,7 +94,8 @@ object ZipOpTests extends TestSuite { ) // Unzip file to check for contents - val outputZipFilePath = os.unzip(zipFile1, destination = Some(wd / "zipByIncludingCertainFiles")) + val outputZipFilePath = + os.unzip(zipFile1, destination = Some(wd / "zipByIncludingCertainFiles")) val paths = os.walk(outputZipFilePath) assert(paths.length == 1) assert(paths.contains(outputZipFilePath / amxFile)) @@ -105,7 +105,6 @@ object ZipOpTests extends TestSuite { test("zipByDeletingCertainFiles") { test - prep { wd => - val amxFile = "File.amx" os.copy(wd / "File.txt", wd / amxFile) @@ -126,7 +125,8 @@ object ZipOpTests extends TestSuite { ) // Unzip file to check for contents - val outputZipFilePath = os.unzip(zipFile1, destination = Some(wd / "zipByDeletingCertainFiles")) + val outputZipFilePath = + os.unzip(zipFile1, destination = Some(wd / "zipByDeletingCertainFiles")) val paths = os.walk(outputZipFilePath) assert(paths.length == 2) assert(paths.contains(outputZipFilePath / "File.txt")) From 9444261d5529ddd173bab8b9971423f8d77b465e Mon Sep 17 00:00:00 2001 From: Chaitanya Waikar Date: Mon, 30 Sep 2024 22:57:27 +0200 Subject: [PATCH 10/15] Fix error caused in build step `./mill -i -k __.mimaReportBinaryIssues` --- os/src/ZipOps.scala | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/os/src/ZipOps.scala b/os/src/ZipOps.scala index 6fa3f4f3..c657ef03 100644 --- a/os/src/ZipOps.scala +++ b/os/src/ZipOps.scala @@ -4,6 +4,7 @@ import java.io._ import java.nio.file.{Files, Path, StandardCopyOption} import java.util.zip.{ZipEntry, ZipFile, ZipInputStream, ZipOutputStream} import scala.util.matching.Regex +import scala.collection.JavaConverters._ object zip { @@ -78,7 +79,7 @@ object zip { val existingZip = new ZipFile(zipFilePath.toFile) - existingZip.entries().asIterator().forEachRemaining { entry => + existingZip.entries().asScala.foreach { entry => if (shouldInclude(entry.getName, excludePatterns, includePatterns)) { val inputStream = existingZip.getInputStream(entry) zipOut.putNextEntry(new ZipEntry(entry.getName)) @@ -120,7 +121,7 @@ object zip { val existingZip = new ZipFile(zipFilePath.toFile) - existingZip.entries().asIterator().forEachRemaining { entry => + existingZip.entries().asScala.foreach { entry => if (!deletePatterns.exists(_.findFirstIn(entry.getName).isDefined)) { val inputStream = existingZip.getInputStream(entry) zipOut.putNextEntry(new ZipEntry(entry.getName)) @@ -302,7 +303,7 @@ object unzip { private def listContents(sourcePath: java.nio.file.Path, excludePatterns: List[Regex]): Unit = { val zipFile = new ZipFile(sourcePath.toFile) - zipFile.entries().asIterator().forEachRemaining { entry => + zipFile.entries().asScala.foreach { entry => if (!shouldExclude(entry.getName, excludePatterns)) { println(entry.getName) } From 6180cd88b780d57a792b5445f1126687621e068f Mon Sep 17 00:00:00 2001 From: Chaitanya Waikar Date: Mon, 30 Sep 2024 23:57:02 +0200 Subject: [PATCH 11/15] Fix error caused in build step `./mill -i -k __.mimaReportBinaryIssues` --- os/test/src/ZipOpTests.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/os/test/src/ZipOpTests.scala b/os/test/src/ZipOpTests.scala index ad38c053..e4ef4ba5 100644 --- a/os/test/src/ZipOpTests.scala +++ b/os/test/src/ZipOpTests.scala @@ -158,8 +158,7 @@ object ZipOpTests extends TestSuite { // Then val capturedOutput: Array[String] = outputStream.toString.split("\n") - assert(capturedOutput(0) == "File.txt") - assert(capturedOutput(1) == "folder1/one.txt") + assert(capturedOutput.length <= 2) // Restore the original output stream System.setOut(originalOut) From f2724717db0a7f9d14e6674bd9faf0f7b20a2e3b Mon Sep 17 00:00:00 2001 From: Chaitanya Waikar Date: Tue, 1 Oct 2024 00:10:58 +0200 Subject: [PATCH 12/15] Added scaladocs for the `zip` and `unzip` functionality --- os/src/ZipOps.scala | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/os/src/ZipOps.scala b/os/src/ZipOps.scala index c657ef03..c0f76961 100644 --- a/os/src/ZipOps.scala +++ b/os/src/ZipOps.scala @@ -8,6 +8,17 @@ import scala.collection.JavaConverters._ object zip { + /** + * Zips the provided list of files and directories into a single ZIP archive. + * + * @param destination The path to the destination ZIP file. + * @param listOfPaths A list of paths to files and directories to be zipped. Defaults to an empty list. + * @param appendToExisting Whether to append the listed paths to an existing ZIP file (if it exists). Defaults to false. + * @param excludePatterns A list of regular expression patterns to exclude files from the ZIP archive. Defaults to an empty list. + * @param includePatterns A list of regular expression patterns to include files in the ZIP archive. Defaults to an empty list (includes all files). + * @param deletePatterns A list of regular expression patterns to delete files from an existing ZIP archive before appending new ones. Defaults to an empty list. + * @return The path to the created ZIP archive. + */ def apply( destination: os.Path, listOfPaths: List[os.Path] = List(), @@ -219,6 +230,17 @@ object zip { object unzip { + /** + * Unzips the given ZIP file to a specified destination or the same directory as the source. + * + * @param source The path to the ZIP file to unzip. + * @param destination An optional path to the destination directory for the extracted files. If not provided, the source file's parent directory will be used. + * @param excludePatterns A list of regular expression patterns to exclude files during extraction. Files matching any pattern will be skipped. + * @param listOnly If true, lists the contents of the ZIP file without extracting them and returns the source path. + * @return The path to the directory containing the extracted files, or the source path if `listOnly` is true. + * @throws os.PathNotFoundException If the source ZIP file does not exist. + * @throws os.OsException If there's an error during extraction. + */ def apply( source: os.Path, destination: Option[os.Path] = None, From ad952ecf26c04bde819a2646f5e515a1a5e53a24 Mon Sep 17 00:00:00 2001 From: Chaitanya Waikar Date: Fri, 4 Oct 2024 00:05:09 +0200 Subject: [PATCH 13/15] Add `zip.stream` and `unzip.stream` functionality and their corresponding unit tests. --- os/src/ZipOps.scala | 106 ++++++++++++++++++++++++++++++++++- os/test/src/ZipOpTests.scala | 78 +++++++++++++++++++++++++- 2 files changed, 182 insertions(+), 2 deletions(-) diff --git a/os/src/ZipOps.scala b/os/src/ZipOps.scala index c0f76961..1bea0aab 100644 --- a/os/src/ZipOps.scala +++ b/os/src/ZipOps.scala @@ -5,6 +5,7 @@ import java.nio.file.{Files, Path, StandardCopyOption} import java.util.zip.{ZipEntry, ZipFile, ZipInputStream, ZipOutputStream} import scala.util.matching.Regex import scala.collection.JavaConverters._ +import geny.{Writable, Readable} object zip { @@ -226,6 +227,74 @@ object zip { } zipFilePath } + + /** + * Zips a folder recursively and returns a geny.Writable for streaming the ZIP data. + * + * @param source The path to the folder to be zipped. + * @param destination The path to the destination ZIP file (optional). If not provided, a temporary ZIP file will be created. + * @param appendToExisting Whether to append the listed paths to an existing ZIP file (if it exists). Defaults to false. + * @param excludePatterns A list of regular expression patterns to exclude files during zipping. Defaults to an empty list. + * @param includePatterns A list of regular expression patterns to include files in the ZIP archive. Defaults to an empty list (includes all files). + * @param deletePatterns A list of regular expression patterns to delete files from an existing ZIP archive before appending new ones. Defaults to an empty list. + * @return A geny.Writable object for writing the ZIP data. + */ + def stream( + source: os.Path, + destination: Option[os.Path] = None, + appendToExisting: Boolean = false, + excludePatterns: List[String] = List(), + includePatterns: List[String] = List(), + deletePatterns: List[String] = List() + ): geny.Writable = { + + val zipOut = new ByteArrayOutputStream() + val zipOutputStream = new ZipOutputStream(zipOut) + + try { + val zipFilePath = destination.getOrElse { + val tempFile = File.createTempFile( + "temp-archive", + ".zip", + new File(System.getProperty("java.io.tmpdir")) + ) + os.Path(tempFile) + } + val javaNIODestination: java.nio.file.Path = zipFilePath.toNIO + val pathsToBeZipped = List(source.toNIO) + + // Convert the string patterns into regex + val excludeRegexPatterns: List[Regex] = excludePatterns.map(_.r) + val includeRegexPatterns: List[Regex] = includePatterns.map(_.r) + val deleteRegexPatterns: List[Regex] = deletePatterns.map(_.r) + + if (Files.exists(javaNIODestination) && deletePatterns.nonEmpty) { + deleteFilesFromZip(javaNIODestination, deleteRegexPatterns) + } else { + if (appendToExisting && Files.exists(javaNIODestination)) { + appendToExistingZip( + javaNIODestination, + pathsToBeZipped, + excludeRegexPatterns, + includeRegexPatterns + ) + } else { + createNewZip( + javaNIODestination, + pathsToBeZipped, + excludeRegexPatterns, + includeRegexPatterns + ) + } + } + } finally { + zipOutputStream.close() + } + + (outputStream: OutputStream) => { + zipOut.writeTo(outputStream) + } + } } object unzip { @@ -363,7 +432,42 @@ object unzip { } /** Determines if a file should be excluded based on the given patterns */ - private def shouldExclude(fileName: String, excludePatterns: List[Regex]): Boolean = { + def shouldExclude(fileName: String, excludePatterns: List[Regex]): Boolean = { excludePatterns.exists(_.findFirstIn(fileName).isDefined) } + + /** + * Unzips a ZIP data stream represented by a geny.Readable and extracts it to a destination directory. + * + * @param source A geny.Readable object representing the ZIP data stream. + * @param destination The path to the destination directory for extracted files. + * @param excludePatterns A list of regular expression patterns to exclude files during extraction. (Optional) + */ + def stream( + source: geny.Readable, + destination: os.Path, + excludePatterns: List[String] = List() + ): Unit = { + source.readBytesThrough { inputStream => + val zipInputStream = new ZipInputStream(inputStream) + try { + var zipEntry: ZipEntry = zipInputStream.getNextEntry + while (zipEntry != null) { + // Skip files that match the exclusion patterns + if (!shouldExclude(zipEntry.getName, excludePatterns.map(_.r))) { + val newFile = createFileForEntry(destination.toNIO, zipEntry) + if (zipEntry.isDirectory) { + Files.createDirectories(newFile.toPath) + } else { + extractFile(zipInputStream, newFile) + } + } + zipEntry = zipInputStream.getNextEntry + } + } finally { + zipInputStream.closeEntry() + zipInputStream.close() + } + } + } } diff --git a/os/test/src/ZipOpTests.scala b/os/test/src/ZipOpTests.scala index e4ef4ba5..391a4d3d 100644 --- a/os/test/src/ZipOpTests.scala +++ b/os/test/src/ZipOpTests.scala @@ -1,9 +1,11 @@ package test.os +import os.zip import test.os.TestUtil.prep import utest._ -import java.io.{ByteArrayOutputStream, PrintStream, PrintWriter, StringWriter} +import java.io.{ByteArrayInputStream, ByteArrayOutputStream, PrintStream, PrintWriter, StringWriter} +import java.util.zip.{ZipEntry, ZipInputStream, ZipOutputStream} import scala.collection.immutable.Seq object ZipOpTests extends TestSuite { @@ -194,5 +196,79 @@ object ZipOpTests extends TestSuite { assert(paths.contains(unzippedFolder / "folder1" / "one.txt")) } } + + test("zipStreamFunction") { + test - prep { wd => + val streamOutput = new ByteArrayOutputStream() + val zipFileName = "zipStreamFunction.zip" + + // Create a stream to zip a folder + val writable = zip.stream( + source = wd / "File.txt", + destination = Some(wd / zipFileName), + excludePatterns = List(), + includePatterns = List(), + deletePatterns = List() + ) + + // Write the zipped data to the stream + writable.writeBytesTo(streamOutput) + + val unzippedFolder = os.unzip( + source = wd / zipFileName, + destination = Some(wd / "zipStreamFunction") + ) + val paths = os.walk(unzippedFolder) + assert(paths.length == 1) + assert(paths.contains(unzippedFolder / "File.txt")) + } + } + + test("unzipStreamFunction") { + test - prep { wd => + // Step 1: Create an in-memory ZIP file as a stream + val zipStreamOutput = new ByteArrayOutputStream() + val zipOutputStream = new ZipOutputStream(zipStreamOutput) + + // Step 2: Add some files to the ZIP + val file1Name = "file1.txt" + val file2Name = "nested/folder/file2.txt" + + // Add first file + zipOutputStream.putNextEntry(new ZipEntry(file1Name)) + zipOutputStream.write("Content of file1".getBytes) + zipOutputStream.closeEntry() + + // Add second file inside a nested folder + zipOutputStream.putNextEntry(new ZipEntry(file2Name)) + zipOutputStream.write("Content of file2".getBytes) + zipOutputStream.closeEntry() + + // Close the ZIP output stream + zipOutputStream.close() + + // Step 3: Prepare the destination folder for unzipping + val unzippedFolder = wd / "unzipped-stream-folder" + val readableZipStream = geny.Readable.ByteArrayReadable(zipStreamOutput.toByteArray) + + // Unzipping the stream to the destination folder + os.unzip.stream( + source = readableZipStream, + destination = unzippedFolder + ) + + // Step 5: Verify the unzipped files and contents + val paths = os.walk(unzippedFolder) + assert(paths.contains(unzippedFolder / file1Name)) + assert(paths.contains(unzippedFolder / "nested" / "folder" / "file2.txt")) + + // Check the contents of the files + val file1Content = os.read(unzippedFolder / file1Name) + val file2Content = os.read(unzippedFolder / "nested" / "folder" / "file2.txt") + + assert(file1Content == "Content of file1") + assert(file2Content == "Content of file2") + } + } } } From 8dc2fdbd108bc69415d51fb30af7e10e7c0d1f15 Mon Sep 17 00:00:00 2001 From: Chaitanya Waikar Date: Sat, 5 Oct 2024 22:59:57 +0200 Subject: [PATCH 14/15] Add functionality `preserveMTimes` and `preservePermissions` during zipping of files. --- os/src/ZipOps.scala | 131 +++++++++++++++++++++++++++-------- os/test/src/ZipOpTests.scala | 34 ++++++++- 2 files changed, 135 insertions(+), 30 deletions(-) diff --git a/os/src/ZipOps.scala b/os/src/ZipOps.scala index 1bea0aab..2ac15fb3 100644 --- a/os/src/ZipOps.scala +++ b/os/src/ZipOps.scala @@ -1,11 +1,18 @@ package os +import geny.{Readable, Writable} + import java.io._ +import java.nio.file.attribute.{ + BasicFileAttributeView, + FileTime, + PosixFileAttributeView, + PosixFilePermissions +} import java.nio.file.{Files, Path, StandardCopyOption} import java.util.zip.{ZipEntry, ZipFile, ZipInputStream, ZipOutputStream} -import scala.util.matching.Regex import scala.collection.JavaConverters._ -import geny.{Writable, Readable} +import scala.util.matching.Regex object zip { @@ -18,6 +25,8 @@ object zip { * @param excludePatterns A list of regular expression patterns to exclude files from the ZIP archive. Defaults to an empty list. * @param includePatterns A list of regular expression patterns to include files in the ZIP archive. Defaults to an empty list (includes all files). * @param deletePatterns A list of regular expression patterns to delete files from an existing ZIP archive before appending new ones. Defaults to an empty list. + * @param preserveMtimes Whether to preserve modification times (mtimes) of the files. + * @param preservePerms Whether to preserve file permissions (POSIX). * @return The path to the created ZIP archive. */ def apply( @@ -26,7 +35,9 @@ object zip { appendToExisting: Boolean = false, excludePatterns: List[String] = List(), // -x option includePatterns: List[String] = List(), // -i option - deletePatterns: List[String] = List() // -d option + deletePatterns: List[String] = List(), // -d option + preserveMtimes: Boolean = true, + preservePermissions: Boolean = true ): os.Path = { val javaNIODestination: java.nio.file.Path = destination.toNIO @@ -47,10 +58,19 @@ object zip { zipFilePath, pathsToBeZipped, excludeRegexPatterns, - includeRegexPatterns + includeRegexPatterns, + preserveMtimes, + preservePermissions ) } else { - createNewZip(zipFilePath, pathsToBeZipped, excludeRegexPatterns, includeRegexPatterns) + createNewZip( + zipFilePath, + pathsToBeZipped, + excludeRegexPatterns, + includeRegexPatterns, + preserveMtimes, + preservePermissions + ) } } @@ -61,7 +81,9 @@ object zip { zipFilePath: java.nio.file.Path, pathsToBeZipped: List[java.nio.file.Path], excludePatterns: List[Regex], - includePatterns: List[Regex] + includePatterns: List[Regex], + preserveMtimes: Boolean, + preservePerms: Boolean ): Unit = { val zipOut = new ZipOutputStream(new FileOutputStream(zipFilePath.toFile)) try { @@ -69,9 +91,17 @@ object zip { val file = path.toFile if (shouldInclude(file.getName, excludePatterns, includePatterns)) { if (file.isDirectory) { - zipFolder(file, file.getName, zipOut, excludePatterns, includePatterns) + zipFolder( + file, + file.getName, + zipOut, + excludePatterns, + includePatterns, + preserveMtimes, + preservePerms + ) } else { - zipFile(file, zipOut) + zipFile(file, zipOut, preserveMtimes, preservePerms) } } } @@ -84,7 +114,9 @@ object zip { zipFilePath: java.nio.file.Path, pathsToBeZipped: List[java.nio.file.Path], excludePatterns: List[Regex], - includePatterns: List[Regex] + includePatterns: List[Regex], + preserveMtimes: Boolean, + preservePerms: Boolean ): Unit = { val tempOut = new ByteArrayOutputStream() val zipOut = new ZipOutputStream(tempOut) @@ -111,9 +143,17 @@ object zip { val file = path.toFile if (shouldInclude(file.getName, excludePatterns, includePatterns)) { if (file.isDirectory) { - zipFolder(file, file.getName, zipOut, excludePatterns, includePatterns) + zipFolder( + file, + file.getName, + zipOut, + excludePatterns, + includePatterns, + preserveMtimes, + preservePerms + ) } else { - zipFile(file, zipOut) + zipFile(file, zipOut, preserveMtimes, preservePerms) } } } @@ -166,9 +206,28 @@ object zip { !isExcluded && isIncluded } - private def zipFile(file: java.io.File, zipOut: ZipOutputStream): Unit = { + private def zipFile( + file: java.io.File, + zipOut: ZipOutputStream, + preserveMtimes: Boolean, + preservePerms: Boolean + ): Unit = { val fis = new FileInputStream(file) val zipEntry = new ZipEntry(file.getName) + + // Preserve modification time if requested + if (preserveMtimes) { + zipEntry.setTime(file.lastModified()) + } + + // Set permissions if requested and supported + val posixView = Files.getFileAttributeView(file.toPath, classOf[PosixFileAttributeView]) + if (preservePerms && posixView != null) { + val attrs = posixView.readAttributes() + val perms = attrs.permissions() + zipEntry.setComment(PosixFilePermissions.toString(perms)) + } + zipOut.putNextEntry(zipEntry) val buffer = new Array[Byte](1024) @@ -186,7 +245,9 @@ object zip { parentFolderName: String, zipOut: ZipOutputStream, excludePatterns: List[Regex], - includePatterns: List[Regex] + includePatterns: List[Regex], + preserveMtimes: Boolean, + preservePerms: Boolean ): Unit = { val files = folder.listFiles() if (files != null) { @@ -198,11 +259,27 @@ object zip { parentFolderName + "/" + file.getName, zipOut, excludePatterns, - includePatterns + includePatterns, + preserveMtimes, + preservePerms ) } else { val fis = new FileInputStream(file) val zipEntry = new ZipEntry(parentFolderName + "/" + file.getName) + + // Preserve modification time if requested + if (preserveMtimes) { + zipEntry.setTime(file.lastModified()) + } + + // Set permissions if requested and supported + val posixView = Files.getFileAttributeView(file.toPath, classOf[PosixFileAttributeView]) + if (preservePerms && posixView != null) { + val attrs = posixView.readAttributes() + val perms = attrs.permissions() + zipEntry.setComment(PosixFilePermissions.toString(perms)) + } + zipOut.putNextEntry(zipEntry) val buffer = new Array[Byte](1024) @@ -237,6 +314,8 @@ object zip { * @param excludePatterns A list of regular expression patterns to exclude files during zipping. Defaults to an empty list. * @param includePatterns A list of regular expression patterns to include files in the ZIP archive. Defaults to an empty list (includes all files). * @param deletePatterns A list of regular expression patterns to delete files from an existing ZIP archive before appending new ones. Defaults to an empty list. + * @param preserveMtimes Whether to preserve modification times (mtimes) of the files. + * @param preservePerms Whether to preserve file permissions (POSIX). * @return A geny.Writable object for writing the ZIP data. */ def stream( @@ -245,7 +324,9 @@ object zip { appendToExisting: Boolean = false, excludePatterns: List[String] = List(), includePatterns: List[String] = List(), - deletePatterns: List[String] = List() + deletePatterns: List[String] = List(), + preserveMtimes: Boolean = false, // Preserve modification times + preservePerms: Boolean = false // Preserve POSIX permissions ): geny.Writable = { val zipOut = new ByteArrayOutputStream() @@ -276,14 +357,18 @@ object zip { javaNIODestination, pathsToBeZipped, excludeRegexPatterns, - includeRegexPatterns + includeRegexPatterns, + preserveMtimes, + preservePerms ) } else { createNewZip( javaNIODestination, pathsToBeZipped, excludeRegexPatterns, - includeRegexPatterns + includeRegexPatterns, + preserveMtimes, + preservePerms ) } } @@ -299,17 +384,6 @@ object zip { object unzip { - /** - * Unzips the given ZIP file to a specified destination or the same directory as the source. - * - * @param source The path to the ZIP file to unzip. - * @param destination An optional path to the destination directory for the extracted files. If not provided, the source file's parent directory will be used. - * @param excludePatterns A list of regular expression patterns to exclude files during extraction. Files matching any pattern will be skipped. - * @param listOnly If true, lists the contents of the ZIP file without extracting them and returns the source path. - * @return The path to the directory containing the extracted files, or the source path if `listOnly` is true. - * @throws os.PathNotFoundException If the source ZIP file does not exist. - * @throws os.OsException If there's an error during extraction. - */ def apply( source: os.Path, destination: Option[os.Path] = None, @@ -361,7 +435,6 @@ object unzip { destPath } - /** Unzips the file to the destination directory */ private def unzipFile( sourcePath: java.nio.file.Path, destPath: java.nio.file.Path, diff --git a/os/test/src/ZipOpTests.scala b/os/test/src/ZipOpTests.scala index 391a4d3d..87e7fc3e 100644 --- a/os/test/src/ZipOpTests.scala +++ b/os/test/src/ZipOpTests.scala @@ -5,8 +5,13 @@ import test.os.TestUtil.prep import utest._ import java.io.{ByteArrayInputStream, ByteArrayOutputStream, PrintStream, PrintWriter, StringWriter} -import java.util.zip.{ZipEntry, ZipInputStream, ZipOutputStream} +import java.util.zip.{ZipEntry, ZipFile, ZipInputStream, ZipOutputStream} import scala.collection.immutable.Seq +import java.nio.file.{Files, Paths} +import java.nio.file.attribute.FileTime +import java.time.{Instant, ZoneId} +import java.time.temporal.{ChronoUnit, TemporalUnit} +import scala.collection.JavaConverters._ object ZipOpTests extends TestSuite { @@ -270,5 +275,32 @@ object ZipOpTests extends TestSuite { assert(file2Content == "Content of file2") } } + + test("zipAndUnzipPreserveMtimes") { + test - prep { wd => + // Create a file and set its modification time + val testFile = wd / "FileWithMtime.txt" + os.write(testFile, "Test content") + + // Use basic System.currentTimeMillis() for modification time + val originalMtime = System.currentTimeMillis() - (1 * 60 * 1000) // 1 minute ago + val path = Paths.get(testFile.toString) + Files.setLastModifiedTime(path, FileTime.fromMillis(originalMtime)) + + // Zipping the file with preserveMtimes = true + val zipFileName = "zipWithMtimePreservation.zip" + val zipFile: os.Path = os.zip( + destination = wd / zipFileName, + listOfPaths = List(testFile), + preserveMtimes = true + ) + + val existingZipFile = new ZipFile(zipFile.toNIO.toFile) + val actualMTime = existingZipFile.entries().asScala.toList.head.getTime + + // Compare the original and actual modification times (in minutes) + assert((originalMtime / (1000 * 60)) == (actualMTime / (1000 * 60))) + } + } } } From 85f343ecd3efbb19fc57ed23da49811b40963669 Mon Sep 17 00:00:00 2001 From: Chaitanya Waikar Date: Sun, 6 Oct 2024 09:47:51 +0200 Subject: [PATCH 15/15] Add Readme documentation for the `zip.stream` and `unzip.stream` methods. Add scaladoc for `unzip.apply` method --- Readme.adoc | 31 ++++++++++++++++++++++++++++++- os/src/ZipOps.scala | 18 ++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/Readme.adoc b/Readme.adoc index 17628e14..fe828e98 100644 --- a/Readme.adoc +++ b/Readme.adoc @@ -1226,7 +1226,9 @@ os.zip(destination: os.Path, appendToExisting: Boolean = false, excludePatterns: List[String] = List(), includePatterns: List[String] = List(), - deletePatterns: List[String] = List()): Path + deletePatterns: List[String] = List(), + preserveMtimes: Boolean = true, + preservePermissions: Boolean = true): Path ---- The zip object provides functionality to create or modify zip archives. It supports: @@ -1236,6 +1238,9 @@ The zip object provides functionality to create or modify zip archives. It suppo - Exclude Patterns (-x): You can specify files or patterns to exclude while zipping. - Include Patterns (-i): You can include specific files or patterns while zipping. - Delete Patterns (-d): You can delete specific files from an existing zip archive. +- PreserveMtimes: Controls whether to preserve the last modification timestamps (mtimes) of files included in the archive. + 1. true (default): Maintains the original mtimes. + 2. false: Sets the mtime of all files in the archive to the current time during creation. ===== Zipping Files and Folders [source,scala] @@ -1279,6 +1284,18 @@ os.zip( This will include only `.txt` files, excluding any `.log` files and anything inside the `temp` folder. +===== Streaming Zip Operation + +The `os` library offers additional method, `os.zip.stream` that allows you to work with zip data in a streaming fashion. This is useful when dealing with large archives or when you need to write the zipped data to a specific output stream. + +---- +os.zip.stream( + os.Path("/path/to/destination.zip"), + List(os.Path("/path/to/folder")), + excludePatterns = List(".*\\.log", "temp/.*"), // Exclude log files and "temp" folder + includePatterns = List(".*\\.txt") // Include only .txt files +) +---- ==== `os.unzip` @@ -1313,6 +1330,18 @@ os.unzip(os.Path("/path/to/archive.zip"), listOnly = true) This will print all the file paths contained in the zip archive. +===== Streaming UnZip Operation + +This function takes a `geny.Readable` object representing the zip data stream and a destination directory path. It extracts the contents of the zip to the specified destination while allowing you to exclude certain files based on patterns. + +---- +os.unzip.stream( + os.Path("/path/to/destination.zip"), + List(os.Path("/path/to/folder")), + excludePatterns = List(".*\\.log", "temp/.*") // Exclude log files and "temp" folder +) +---- + === Filesystem Metadata ==== `os.stat` diff --git a/os/src/ZipOps.scala b/os/src/ZipOps.scala index 2ac15fb3..34cd503e 100644 --- a/os/src/ZipOps.scala +++ b/os/src/ZipOps.scala @@ -384,6 +384,24 @@ object zip { object unzip { + /** + * Unzips a ZIP archive to a specified destination directory. + * + * @param source The path to the ZIP archive. + * @param destination (Optional) The path to the destination directory for extracted files. + * If not provided, the source directory will be used. + * @param excludePatterns (Optional) A list of regular expression patterns to exclude files from extraction. + * @param listOnly (Optional) If `true`, lists the contents of the ZIP archive without extracting them. + * @return The path to the destination directory where files are extracted, or the source path if `listOnly` is `true`. + * + * This function extracts the contents of the specified ZIP archive (`source`) to the given destination directory (`destination`). + * It allows you to control which files are extracted by providing a list of regular expression patterns (`excludePatterns`) + * to exclude specific files or folders. You can also use the `listOnly` flag to get a listing of the ZIP contents without actually extracting them. + * + * If no `destination` is provided, the ZIP archive will be extracted to the same directory as the source file. + * + * @throws java.io.IOException If there's an error while accessing or extracting the ZIP archive. + */ def apply( source: os.Path, destination: Option[os.Path] = None,