From 988da17b66a8861acdba31cd9feeae1651c9e03c Mon Sep 17 00:00:00 2001 From: "Agarwal, Udit" Date: Mon, 2 Sep 2024 13:51:45 -0700 Subject: [PATCH 1/3] [llvm] Add API to get decompressed size of a zstd compressed buffer. --- llvm/include/llvm/Support/Compression.h | 2 ++ llvm/lib/Support/Compression.cpp | 18 ++++++++++++++++++ llvm/unittests/Support/CompressionTest.cpp | 9 ++++++++- 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/llvm/include/llvm/Support/Compression.h b/llvm/include/llvm/Support/Compression.h index 2a8da9e96d356..da7b6f88d955b 100644 --- a/llvm/include/llvm/Support/Compression.h +++ b/llvm/include/llvm/Support/Compression.h @@ -71,6 +71,8 @@ Error decompress(ArrayRef Input, uint8_t *Output, Error decompress(ArrayRef Input, SmallVectorImpl &Output, size_t UncompressedSize); +// Get the size of the decompressed data. +Error getDecompressedSize(ArrayRef Input, size_t &UncompressedSize); } // End of namespace zstd enum class Format { diff --git a/llvm/lib/Support/Compression.cpp b/llvm/lib/Support/Compression.cpp index badaf68ab59cd..ab012dfb5612b 100644 --- a/llvm/lib/Support/Compression.cpp +++ b/llvm/lib/Support/Compression.cpp @@ -224,6 +224,20 @@ Error zstd::decompress(ArrayRef Input, return E; } +Error zstd::getDecompressedSize(ArrayRef Input, + size_t &UncompressedSize) { + + unsigned long long Res = ZSTD_getFrameContentSize(Input.data(), Input.size()); + + // ZSTD_getFrameContentSize returns unsigned long long, but the size + // of uncompressed data should be bounded by size_t. + UncompressedSize = static_cast(Res); + + return ZSTD_isError(Res) ? make_error(ZSTD_getErrorName(Res), + inconvertibleErrorCode()) + : Error::success(); +} + #else bool zstd::isAvailable() { return false; } void zstd::compress(ArrayRef Input, @@ -240,4 +254,8 @@ Error zstd::decompress(ArrayRef Input, size_t UncompressedSize) { llvm_unreachable("zstd::decompress is unavailable"); } +Error zstd::getDecompressedSize(ArrayRef Input, + size_t &UncompressedSize) { + llvm_unreachable("zstd::getDecompressedSize is unavailable"); +} #endif diff --git a/llvm/unittests/Support/CompressionTest.cpp b/llvm/unittests/Support/CompressionTest.cpp index 5d326cafbe3a1..058be206d28b1 100644 --- a/llvm/unittests/Support/CompressionTest.cpp +++ b/llvm/unittests/Support/CompressionTest.cpp @@ -73,8 +73,15 @@ static void testZstdCompression(StringRef Input, int Level) { SmallVector Uncompressed; zstd::compress(arrayRefFromStringRef(Input), Compressed, Level); + // Check that getDecompressedSize returns the size of the original buffer. + size_t DecompressedSize; + Error E = zstd::getDecompressedSize(Compressed, DecompressedSize); + EXPECT_FALSE(std::move(E)); + + EXPECT_EQ(DecompressedSize, Input.size()); + // Check that uncompressed buffer is the same as original. - Error E = zstd::decompress(Compressed, Uncompressed, Input.size()); + E = zstd::decompress(Compressed, Uncompressed, Input.size()); EXPECT_FALSE(std::move(E)); EXPECT_EQ(Input, toStringRef(Uncompressed)); From c9046d7b10ec896ab3b6bb9a7f6b5452f62feb4c Mon Sep 17 00:00:00 2001 From: "Agarwal, Udit" Date: Mon, 19 May 2025 15:07:13 -0700 Subject: [PATCH 2/3] Use Expected<> instead --- llvm/include/llvm/Support/Compression.h | 3 ++- llvm/lib/Support/Compression.cpp | 13 +++++-------- llvm/unittests/Support/CompressionTest.cpp | 10 ++++------ 3 files changed, 11 insertions(+), 15 deletions(-) diff --git a/llvm/include/llvm/Support/Compression.h b/llvm/include/llvm/Support/Compression.h index d6eb812ed9092..64dadf96feb43 100644 --- a/llvm/include/llvm/Support/Compression.h +++ b/llvm/include/llvm/Support/Compression.h @@ -16,6 +16,7 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/Support/Compiler.h" #include "llvm/Support/DataTypes.h" +#include "llvm/Support/Error.h" namespace llvm { template class SmallVectorImpl; @@ -75,7 +76,7 @@ LLVM_ABI Error decompress(ArrayRef Input, size_t UncompressedSize); // Get the size of the decompressed data. -Error getDecompressedSize(ArrayRef Input, size_t &UncompressedSize); +Expected getDecompressedSize(ArrayRef Input); } // End of namespace zstd enum class Format { diff --git a/llvm/lib/Support/Compression.cpp b/llvm/lib/Support/Compression.cpp index 4ea9fbf4b4f49..95bb1ec928c3f 100644 --- a/llvm/lib/Support/Compression.cpp +++ b/llvm/lib/Support/Compression.cpp @@ -225,18 +225,15 @@ Error zstd::decompress(ArrayRef Input, return E; } -Error zstd::getDecompressedSize(ArrayRef Input, - size_t &UncompressedSize) { +Expected zstd::getDecompressedSize(ArrayRef Input) { unsigned long long Res = ZSTD_getFrameContentSize(Input.data(), Input.size()); - // ZSTD_getFrameContentSize returns unsigned long long, but the size - // of uncompressed data should be bounded by size_t. - UncompressedSize = static_cast(Res); + if (ZSTD_isError(Res)) + return make_error(ZSTD_getErrorName(Res), + inconvertibleErrorCode()); - return ZSTD_isError(Res) ? make_error(ZSTD_getErrorName(Res), - inconvertibleErrorCode()) - : Error::success(); + return Res; } #else diff --git a/llvm/unittests/Support/CompressionTest.cpp b/llvm/unittests/Support/CompressionTest.cpp index 058be206d28b1..4ad1d3ba4fec6 100644 --- a/llvm/unittests/Support/CompressionTest.cpp +++ b/llvm/unittests/Support/CompressionTest.cpp @@ -74,14 +74,12 @@ static void testZstdCompression(StringRef Input, int Level) { zstd::compress(arrayRefFromStringRef(Input), Compressed, Level); // Check that getDecompressedSize returns the size of the original buffer. - size_t DecompressedSize; - Error E = zstd::getDecompressedSize(Compressed, DecompressedSize); - EXPECT_FALSE(std::move(E)); - - EXPECT_EQ(DecompressedSize, Input.size()); + Expected DecompressedSize = zstd::getDecompressedSize(Compressed); + EXPECT_FALSE(!DecompressedSize); + EXPECT_EQ(DecompressedSize.get(), Input.size()); // Check that uncompressed buffer is the same as original. - E = zstd::decompress(Compressed, Uncompressed, Input.size()); + Error E = zstd::decompress(Compressed, Uncompressed, Input.size()); EXPECT_FALSE(std::move(E)); EXPECT_EQ(Input, toStringRef(Uncompressed)); From cf5919d83335b68ccc6ee79060bdcf6a606b2b7a Mon Sep 17 00:00:00 2001 From: "Agarwal, Udit" Date: Mon, 19 May 2025 15:36:22 -0700 Subject: [PATCH 3/3] Fix windows failure for zlib --- llvm/lib/Support/Compression.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/llvm/lib/Support/Compression.cpp b/llvm/lib/Support/Compression.cpp index 95bb1ec928c3f..ffa073d0be932 100644 --- a/llvm/lib/Support/Compression.cpp +++ b/llvm/lib/Support/Compression.cpp @@ -252,8 +252,7 @@ Error zstd::decompress(ArrayRef Input, size_t UncompressedSize) { llvm_unreachable("zstd::decompress is unavailable"); } -Error zstd::getDecompressedSize(ArrayRef Input, - size_t &UncompressedSize) { +Expected zstd::getDecompressedSize(ArrayRef Input) { llvm_unreachable("zstd::getDecompressedSize is unavailable"); } #endif