diff --git a/fileutil/compressor_interface.go b/fileutil/compressor_interface.go index 5a69543d..573c02cd 100644 --- a/fileutil/compressor_interface.go +++ b/fileutil/compressor_interface.go @@ -4,16 +4,19 @@ type CompressorOptions struct { SameOwner bool PathInArchive string StripComponents int + NoCompression bool } type Compressor interface { // CompressFilesInDir returns path to a compressed file - CompressFilesInDir(dir string) (path string, err error) + CompressFilesInDir(dir string, options CompressorOptions) (path string, err error) - CompressSpecificFilesInDir(dir string, files []string) (path string, err error) + CompressSpecificFilesInDir(dir string, files []string, options CompressorOptions) (path string, err error) DecompressFileToDir(path string, dir string, options CompressorOptions) (err error) + IsNonCompressedTarball(path string) (bool, error) + // CleanUp cleans up compressed file after it was used CleanUp(path string) error } diff --git a/fileutil/fakes/fake_compressor.go b/fileutil/fakes/fake_compressor.go index 38c218ae..a0653417 100644 --- a/fileutil/fakes/fake_compressor.go +++ b/fileutil/fakes/fake_compressor.go @@ -6,12 +6,14 @@ import ( type FakeCompressor struct { CompressFilesInDirDir string + CompressFilesInDirOptions boshcmd.CompressorOptions CompressFilesInDirTarballPath string CompressFilesInDirErr error CompressFilesInDirCallBack func() CompressSpecificFilesInDirDir string CompressSpecificFilesInDirFiles []string + CompressSpecificFilesInDirOptions boshcmd.CompressorOptions CompressSpecificFilesInDirTarballPath string CompressSpecificFilesInDirErr error CompressSpecificFilesInDirCallBack func() @@ -24,15 +26,18 @@ type FakeCompressor struct { CleanUpTarballPath string CleanUpErr error + + IsNonCompressedResult bool + IsNonCompressedErr error } func NewFakeCompressor() *FakeCompressor { return &FakeCompressor{} } -func (fc *FakeCompressor) CompressFilesInDir(dir string) (string, error) { +func (fc *FakeCompressor) CompressFilesInDir(dir string, options boshcmd.CompressorOptions) (string, error) { fc.CompressFilesInDirDir = dir - + fc.CompressFilesInDirOptions = options if fc.CompressFilesInDirCallBack != nil { fc.CompressFilesInDirCallBack() } @@ -40,10 +45,10 @@ func (fc *FakeCompressor) CompressFilesInDir(dir string) (string, error) { return fc.CompressFilesInDirTarballPath, fc.CompressFilesInDirErr } -func (fc *FakeCompressor) CompressSpecificFilesInDir(dir string, files []string) (string, error) { +func (fc *FakeCompressor) CompressSpecificFilesInDir(dir string, files []string, options boshcmd.CompressorOptions) (string, error) { fc.CompressSpecificFilesInDirDir = dir fc.CompressSpecificFilesInDirFiles = files - + fc.CompressSpecificFilesInDirOptions = options if fc.CompressSpecificFilesInDirCallBack != nil { fc.CompressSpecificFilesInDirCallBack() } @@ -63,6 +68,10 @@ func (fc *FakeCompressor) DecompressFileToDir(tarballPath string, dir string, op return fc.DecompressFileToDirErr } +func (fc *FakeCompressor) IsNonCompressedTarball(path string) (bool, error) { + return fc.IsNonCompressedResult, fc.IsNonCompressedErr +} + func (fc *FakeCompressor) CleanUp(tarballPath string) error { fc.CleanUpTarballPath = tarballPath return fc.CleanUpErr diff --git a/fileutil/tarball_compressor.go b/fileutil/tarball_compressor.go index 997f1f3c..3df17d71 100644 --- a/fileutil/tarball_compressor.go +++ b/fileutil/tarball_compressor.go @@ -1,13 +1,24 @@ package fileutil import ( + "bytes" "fmt" + "os" "runtime" bosherr "github.com/cloudfoundry/bosh-utils/errors" boshsys "github.com/cloudfoundry/bosh-utils/system" ) +var ( + gzipMagic = []byte{0x1f, 0x8b} + bzip2Magic = []byte{0x42, 0x5a, 0x68} // "BZh" + xzMagic = []byte{0xfd, 0x37, 0x7a, 0x58, 0x5a, 0x00} + zstdMagic = []byte{0x28, 0xb5, 0x2f, 0xfd} + ustarMagic = []byte("ustar") + ustarOffset = 257 // Offset of the TAR magic string in the file +) + type tarballCompressor struct { cmdRunner boshsys.CmdRunner fs boshsys.FileSystem @@ -20,11 +31,11 @@ func NewTarballCompressor( return tarballCompressor{cmdRunner: cmdRunner, fs: fs} } -func (c tarballCompressor) CompressFilesInDir(dir string) (string, error) { - return c.CompressSpecificFilesInDir(dir, []string{"."}) +func (c tarballCompressor) CompressFilesInDir(dir string, options CompressorOptions) (string, error) { + return c.CompressSpecificFilesInDir(dir, []string{"."}, options) } -func (c tarballCompressor) CompressSpecificFilesInDir(dir string, files []string) (string, error) { +func (c tarballCompressor) CompressSpecificFilesInDir(dir string, files []string, options CompressorOptions) (string, error) { tarball, err := c.fs.TempFile("bosh-platform-disk-TarballCompressor-CompressSpecificFilesInDir") if err != nil { return "", bosherr.WrapError(err, "Creating temporary file for tarball") @@ -34,7 +45,10 @@ func (c tarballCompressor) CompressSpecificFilesInDir(dir string, files []string tarballPath := tarball.Name() - args := []string{"-czf", tarballPath, "-C", dir} + args := []string{"-cf", tarballPath, "-C", dir} + if !options.NoCompression { + args = append(args, "-z") + } if runtime.GOOS == "darwin" { args = append([]string{"--no-mac-metadata"}, args...) } @@ -61,7 +75,7 @@ func (c tarballCompressor) DecompressFileToDir(tarballPath string, dir string, o if err != nil { return bosherr.WrapError(err, "Resolving tarball path") } - args := []string{sameOwnerOption, "-xzf", resolvedTarballPath, "-C", dir} + args := []string{sameOwnerOption, "-xf", resolvedTarballPath, "-C", dir} if options.StripComponents != 0 { args = append(args, fmt.Sprintf("--strip-components=%d", options.StripComponents)) } @@ -77,6 +91,38 @@ func (c tarballCompressor) DecompressFileToDir(tarballPath string, dir string, o return nil } +func (c tarballCompressor) IsNonCompressedTarball(path string) (bool, error) { + f, err := c.fs.OpenFile(path, os.O_RDONLY, 0644) + if err != nil { + return false, fmt.Errorf("could not open file: %w", err) + } + defer f.Close() + + // Read the first 512 bytes to check both compression headers and the TAR header. + // Ignore the error from reading a partial buffer, which is fine for short files. + buffer := make([]byte, 512) + _, _ = f.Read(buffer) + + // 1. Check for compression first. + if bytes.HasPrefix(buffer, gzipMagic) || + bytes.HasPrefix(buffer, bzip2Magic) || + bytes.HasPrefix(buffer, xzMagic) || + bytes.HasPrefix(buffer, zstdMagic) { + return false, nil + } + + // 2. If NOT compressed, check for the TAR magic string at its specific offset. + // Ensure the buffer is long enough to contain the TAR header magic string. + if len(buffer) > ustarOffset+len(ustarMagic) { + magicBytes := buffer[ustarOffset : ustarOffset+len(ustarMagic)] + if bytes.Equal(magicBytes, ustarMagic) { + return true, nil + } + } + + return false, nil +} + func (c tarballCompressor) CleanUp(tarballPath string) error { return c.fs.RemoveAll(tarballPath) } diff --git a/fileutil/tarball_compressor_test.go b/fileutil/tarball_compressor_test.go index 2a80682e..d281dfb9 100644 --- a/fileutil/tarball_compressor_test.go +++ b/fileutil/tarball_compressor_test.go @@ -53,7 +53,7 @@ var _ = Describe("tarballCompressor", func() { defer os.Remove(symlinkPath) - tgzName, err := compressor.CompressFilesInDir(testAssetsFixtureDir) + tgzName, err := compressor.CompressFilesInDir(testAssetsFixtureDir, CompressorOptions{}) Expect(err).ToNot(HaveOccurred()) defer os.Remove(tgzName) @@ -94,6 +94,30 @@ var _ = Describe("tarballCompressor", func() { Expect(err).ToNot(HaveOccurred()) Expect(content).To(ContainSubstring("this is other app stdout")) }) + + It("uses NoCompression option to create uncompressed tarball", func() { + cmdRunner := fakesys.NewFakeCmdRunner() + compressor := NewTarballCompressor(cmdRunner, fs) + + tgzName, err := compressor.CompressFilesInDir(testAssetsFixtureDir, CompressorOptions{NoCompression: true}) + Expect(err).ToNot(HaveOccurred()) + defer os.Remove(tgzName) + + Expect(1).To(Equal(len(cmdRunner.RunCommands))) + Expect(cmdRunner.RunCommands[0]).ToNot(ContainElement("-z")) + }) + + It("uses compression by default when NoCompression is false", func() { + cmdRunner := fakesys.NewFakeCmdRunner() + compressor := NewTarballCompressor(cmdRunner, fs) + + tgzName, err := compressor.CompressFilesInDir(testAssetsFixtureDir, CompressorOptions{NoCompression: false}) + Expect(err).ToNot(HaveOccurred()) + defer os.Remove(tgzName) + + Expect(1).To(Equal(len(cmdRunner.RunCommands))) + Expect(cmdRunner.RunCommands[0]).To(ContainElement("-z")) + }) }) Describe("CompressSpecificFilesInDir", func() { @@ -104,7 +128,7 @@ var _ = Describe("tarballCompressor", func() { "some_directory", "app.stderr.log", } - tgzName, err := compressor.CompressSpecificFilesInDir(srcDir, files) + tgzName, err := compressor.CompressSpecificFilesInDir(srcDir, files, CompressorOptions{}) Expect(err).ToNot(HaveOccurred()) defer os.Remove(tgzName) @@ -182,7 +206,7 @@ var _ = Describe("tarballCompressor", func() { Expect(cmdRunner.RunCommands[0]).To(Equal( []string{ "tar", "--no-same-owner", - "-xzf", tarballPath, + "-xf", tarballPath, "-C", dstDir, }, )) @@ -204,7 +228,7 @@ var _ = Describe("tarballCompressor", func() { Expect(cmdRunner.RunCommands[0]).To(Equal( []string{ "tar", "--same-owner", - "-xzf", tarballPath, + "-xf", tarballPath, "-C", dstDir, }, )) @@ -222,7 +246,7 @@ var _ = Describe("tarballCompressor", func() { Expect(cmdRunner.RunCommands[0]).To(Equal( []string{ "tar", "--no-same-owner", - "-xzf", tarballPath, + "-xf", tarballPath, "-C", dstDir, "some/path/in/archive", }, @@ -241,7 +265,7 @@ var _ = Describe("tarballCompressor", func() { Expect(cmdRunner.RunCommands[0]).To(Equal( []string{ "tar", "--no-same-owner", - "-xzf", tarballPath, + "-xf", tarballPath, "-C", dstDir, "--strip-components=3", }, @@ -249,6 +273,88 @@ var _ = Describe("tarballCompressor", func() { }) }) + Describe("IsNonCompressedTarball", func() { + It("returns true for non-compressed tarball created with NoCompression=true", func() { + tgzName, err := compressor.CompressFilesInDir(testAssetsFixtureDir, CompressorOptions{NoCompression: true}) + Expect(err).ToNot(HaveOccurred()) + defer os.Remove(tgzName) + + result, err := compressor.IsNonCompressedTarball(tgzName) + Expect(err).ToNot(HaveOccurred()) + Expect(result).To(BeTrue()) + }) + + It("returns false for compressed tarball created with NoCompression=false", func() { + tgzName, err := compressor.CompressFilesInDir(testAssetsFixtureDir, CompressorOptions{NoCompression: false}) + Expect(err).ToNot(HaveOccurred()) + defer os.Remove(tgzName) + + result, err := compressor.IsNonCompressedTarball(tgzName) + Expect(err).ToNot(HaveOccurred()) + Expect(result).To(BeFalse()) + }) + + It("returns false for compressed tarball created with default options", func() { + tgzName, err := compressor.CompressFilesInDir(testAssetsFixtureDir, CompressorOptions{}) + Expect(err).ToNot(HaveOccurred()) + defer os.Remove(tgzName) + + result, err := compressor.IsNonCompressedTarball(tgzName) + Expect(err).ToNot(HaveOccurred()) + Expect(result).To(BeFalse()) + }) + + It("returns error for non-existent file", func() { + result, err := compressor.IsNonCompressedTarball("/nonexistent/file.tar") + Expect(err).To(HaveOccurred()) + Expect(result).To(BeFalse()) + }) + + It("returns error for non-tarball file", func() { + tempFile, err := fs.TempFile("test-non-tarball") + Expect(err).ToNot(HaveOccurred()) + defer os.Remove(tempFile.Name()) + + err = fs.WriteFileString(tempFile.Name(), "This is not a tar file") + Expect(err).ToNot(HaveOccurred()) + + result, err := compressor.IsNonCompressedTarball(tempFile.Name()) + Expect(err).ToNot(HaveOccurred()) + Expect(result).To(BeFalse()) + }) + + It("returns error for empty file", func() { + tempFile, err := fs.TempFile("test-empty-file") + Expect(err).ToNot(HaveOccurred()) + defer os.Remove(tempFile.Name()) + tempFile.Close() + + result, err := compressor.IsNonCompressedTarball(tempFile.Name()) + Expect(err).ToNot(HaveOccurred()) + Expect(result).To(BeFalse()) + }) + + It("correctly identifies tarballs created with CompressSpecificFilesInDir", func() { + files := []string{"app.stdout.log", "app.stderr.log"} + + tgzName, err := compressor.CompressSpecificFilesInDir(testAssetsFixtureDir, files, CompressorOptions{NoCompression: true}) + Expect(err).ToNot(HaveOccurred()) + defer os.Remove(tgzName) + + result, err := compressor.IsNonCompressedTarball(tgzName) + Expect(err).ToNot(HaveOccurred()) + Expect(result).To(BeTrue()) + + tgzName2, err := compressor.CompressSpecificFilesInDir(testAssetsFixtureDir, files, CompressorOptions{NoCompression: false}) + Expect(err).ToNot(HaveOccurred()) + defer os.Remove(tgzName2) + + result2, err := compressor.IsNonCompressedTarball(tgzName2) + Expect(err).ToNot(HaveOccurred()) + Expect(result2).To(BeFalse()) + }) + }) + Describe("CleanUp", func() { It("removes tarball path", func() { fs := fakesys.NewFakeFileSystem()