From 6692359898c6ccae504b634de02203d34137a1cc Mon Sep 17 00:00:00 2001 From: Oleksandr Yermoshenko Date: Tue, 11 May 2021 11:42:06 +0300 Subject: [PATCH 1/9] Added SHA1 implementation from https://github.com/983/SHA1 --- lib/SHA1/sha1.hpp | 255 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 255 insertions(+) create mode 100644 lib/SHA1/sha1.hpp diff --git a/lib/SHA1/sha1.hpp b/lib/SHA1/sha1.hpp new file mode 100644 index 0000000..f0fdbc9 --- /dev/null +++ b/lib/SHA1/sha1.hpp @@ -0,0 +1,255 @@ +#include +#include + +#define SHA1_HEX_SIZE (40 + 1) +#define SHA1_BASE64_SIZE (28 + 1) + +class sha1 { +private: + + void add_byte_dont_count_bits(uint8_t x){ + buf[i++] = x; + + if (i >= sizeof(buf)){ + i = 0; + process_block(buf); + } + } + + static uint32_t rol32(uint32_t x, uint32_t n){ + return (x << n) | (x >> (32 - n)); + } + + static uint32_t make_word(const uint8_t *p){ + return + ((uint32_t)p[0] << 3*8) | + ((uint32_t)p[1] << 2*8) | + ((uint32_t)p[2] << 1*8) | + ((uint32_t)p[3] << 0*8); + } + + void process_block(const uint8_t *ptr){ + const uint32_t c0 = 0x5a827999; + const uint32_t c1 = 0x6ed9eba1; + const uint32_t c2 = 0x8f1bbcdc; + const uint32_t c3 = 0xca62c1d6; + + uint32_t a = state[0]; + uint32_t b = state[1]; + uint32_t c = state[2]; + uint32_t d = state[3]; + uint32_t e = state[4]; + + uint32_t w[16]; + + for (int i = 0; i < 16; i++) w[i] = make_word(ptr + i*4); + +#define SHA1_LOAD(i) w[i&15] = rol32(w[(i+13)&15] ^ w[(i+8)&15] ^ w[(i+2)&15] ^ w[i&15], 1); +#define SHA1_ROUND_0(v,u,x,y,z,i) z += ((u & (x ^ y)) ^ y) + w[i&15] + c0 + rol32(v, 5); u = rol32(u, 30); +#define SHA1_ROUND_1(v,u,x,y,z,i) SHA1_LOAD(i) z += ((u & (x ^ y)) ^ y) + w[i&15] + c0 + rol32(v, 5); u = rol32(u, 30); +#define SHA1_ROUND_2(v,u,x,y,z,i) SHA1_LOAD(i) z += (u ^ x ^ y) + w[i&15] + c1 + rol32(v, 5); u = rol32(u, 30); +#define SHA1_ROUND_3(v,u,x,y,z,i) SHA1_LOAD(i) z += (((u | x) & y) | (u & x)) + w[i&15] + c2 + rol32(v, 5); u = rol32(u, 30); +#define SHA1_ROUND_4(v,u,x,y,z,i) SHA1_LOAD(i) z += (u ^ x ^ y) + w[i&15] + c3 + rol32(v, 5); u = rol32(u, 30); + + SHA1_ROUND_0(a, b, c, d, e, 0); + SHA1_ROUND_0(e, a, b, c, d, 1); + SHA1_ROUND_0(d, e, a, b, c, 2); + SHA1_ROUND_0(c, d, e, a, b, 3); + SHA1_ROUND_0(b, c, d, e, a, 4); + SHA1_ROUND_0(a, b, c, d, e, 5); + SHA1_ROUND_0(e, a, b, c, d, 6); + SHA1_ROUND_0(d, e, a, b, c, 7); + SHA1_ROUND_0(c, d, e, a, b, 8); + SHA1_ROUND_0(b, c, d, e, a, 9); + SHA1_ROUND_0(a, b, c, d, e, 10); + SHA1_ROUND_0(e, a, b, c, d, 11); + SHA1_ROUND_0(d, e, a, b, c, 12); + SHA1_ROUND_0(c, d, e, a, b, 13); + SHA1_ROUND_0(b, c, d, e, a, 14); + SHA1_ROUND_0(a, b, c, d, e, 15); + SHA1_ROUND_1(e, a, b, c, d, 16); + SHA1_ROUND_1(d, e, a, b, c, 17); + SHA1_ROUND_1(c, d, e, a, b, 18); + SHA1_ROUND_1(b, c, d, e, a, 19); + SHA1_ROUND_2(a, b, c, d, e, 20); + SHA1_ROUND_2(e, a, b, c, d, 21); + SHA1_ROUND_2(d, e, a, b, c, 22); + SHA1_ROUND_2(c, d, e, a, b, 23); + SHA1_ROUND_2(b, c, d, e, a, 24); + SHA1_ROUND_2(a, b, c, d, e, 25); + SHA1_ROUND_2(e, a, b, c, d, 26); + SHA1_ROUND_2(d, e, a, b, c, 27); + SHA1_ROUND_2(c, d, e, a, b, 28); + SHA1_ROUND_2(b, c, d, e, a, 29); + SHA1_ROUND_2(a, b, c, d, e, 30); + SHA1_ROUND_2(e, a, b, c, d, 31); + SHA1_ROUND_2(d, e, a, b, c, 32); + SHA1_ROUND_2(c, d, e, a, b, 33); + SHA1_ROUND_2(b, c, d, e, a, 34); + SHA1_ROUND_2(a, b, c, d, e, 35); + SHA1_ROUND_2(e, a, b, c, d, 36); + SHA1_ROUND_2(d, e, a, b, c, 37); + SHA1_ROUND_2(c, d, e, a, b, 38); + SHA1_ROUND_2(b, c, d, e, a, 39); + SHA1_ROUND_3(a, b, c, d, e, 40); + SHA1_ROUND_3(e, a, b, c, d, 41); + SHA1_ROUND_3(d, e, a, b, c, 42); + SHA1_ROUND_3(c, d, e, a, b, 43); + SHA1_ROUND_3(b, c, d, e, a, 44); + SHA1_ROUND_3(a, b, c, d, e, 45); + SHA1_ROUND_3(e, a, b, c, d, 46); + SHA1_ROUND_3(d, e, a, b, c, 47); + SHA1_ROUND_3(c, d, e, a, b, 48); + SHA1_ROUND_3(b, c, d, e, a, 49); + SHA1_ROUND_3(a, b, c, d, e, 50); + SHA1_ROUND_3(e, a, b, c, d, 51); + SHA1_ROUND_3(d, e, a, b, c, 52); + SHA1_ROUND_3(c, d, e, a, b, 53); + SHA1_ROUND_3(b, c, d, e, a, 54); + SHA1_ROUND_3(a, b, c, d, e, 55); + SHA1_ROUND_3(e, a, b, c, d, 56); + SHA1_ROUND_3(d, e, a, b, c, 57); + SHA1_ROUND_3(c, d, e, a, b, 58); + SHA1_ROUND_3(b, c, d, e, a, 59); + SHA1_ROUND_4(a, b, c, d, e, 60); + SHA1_ROUND_4(e, a, b, c, d, 61); + SHA1_ROUND_4(d, e, a, b, c, 62); + SHA1_ROUND_4(c, d, e, a, b, 63); + SHA1_ROUND_4(b, c, d, e, a, 64); + SHA1_ROUND_4(a, b, c, d, e, 65); + SHA1_ROUND_4(e, a, b, c, d, 66); + SHA1_ROUND_4(d, e, a, b, c, 67); + SHA1_ROUND_4(c, d, e, a, b, 68); + SHA1_ROUND_4(b, c, d, e, a, 69); + SHA1_ROUND_4(a, b, c, d, e, 70); + SHA1_ROUND_4(e, a, b, c, d, 71); + SHA1_ROUND_4(d, e, a, b, c, 72); + SHA1_ROUND_4(c, d, e, a, b, 73); + SHA1_ROUND_4(b, c, d, e, a, 74); + SHA1_ROUND_4(a, b, c, d, e, 75); + SHA1_ROUND_4(e, a, b, c, d, 76); + SHA1_ROUND_4(d, e, a, b, c, 77); + SHA1_ROUND_4(c, d, e, a, b, 78); + SHA1_ROUND_4(b, c, d, e, a, 79); + +#undef SHA1_LOAD +#undef SHA1_ROUND_0 +#undef SHA1_ROUND_1 +#undef SHA1_ROUND_2 +#undef SHA1_ROUND_3 +#undef SHA1_ROUND_4 + + state[0] += a; + state[1] += b; + state[2] += c; + state[3] += d; + state[4] += e; + } + +public: + + uint32_t state[5]; + uint8_t buf[64]; + uint32_t i; + uint64_t n_bits; + + sha1(const char *text = NULL): i(0), n_bits(0){ + state[0] = 0x67452301; + state[1] = 0xEFCDAB89; + state[2] = 0x98BADCFE; + state[3] = 0x10325476; + state[4] = 0xC3D2E1F0; + if (text) add(text); + } + + sha1& add(uint8_t x){ + add_byte_dont_count_bits(x); + n_bits += 8; + return *this; + } + + sha1& add(char c){ + return add(*(uint8_t*)&c); + } + + sha1& add(const void *data, uint32_t n){ + if (!data) return *this; + + const uint8_t *ptr = (const uint8_t*)data; + + // fill up block if not full + for (; n && i % sizeof(buf); n--) add(*ptr++); + + // process full blocks + for (; n >= sizeof(buf); n -= sizeof(buf)){ + process_block(ptr); + ptr += sizeof(buf); + n_bits += sizeof(buf) * 8; + } + + // process remaining part of block + for (; n; n--) add(*ptr++); + + return *this; + } + + sha1& add(const char *text){ + if (!text) return *this; + return add(text, strlen(text)); + } + + sha1& finalize(){ + // hashed text ends with 0x80, some padding 0x00 and the length in bits + add_byte_dont_count_bits(0x80); + while (i % 64 != 56) add_byte_dont_count_bits(0x00); + for (int j = 7; j >= 0; j--) add_byte_dont_count_bits(n_bits >> j * 8); + + return *this; + } + + const sha1& print_hex( + char *hex, + bool zero_terminate = true, + const char *alphabet = "0123456789abcdef" + ) const { + // print hex + int k = 0; + for (int i = 0; i < 5; i++){ + for (int j = 7; j >= 0; j--){ + hex[k++] = alphabet[(state[i] >> j * 4) & 0xf]; + } + } + if (zero_terminate) hex[k] = '\0'; + return *this; + } + + const sha1& print_base64(char *base64, bool zero_terminate = true) const { + static const uint8_t *table = (const uint8_t*) + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "abcdefghijklmnopqrstuvwxyz" + "0123456789" + "+/"; + + uint32_t triples[7] = { + ((state[0] & 0xffffff00) >> 1*8), + ((state[0] & 0x000000ff) << 2*8) | ((state[1] & 0xffff0000) >> 2*8), + ((state[1] & 0x0000ffff) << 1*8) | ((state[2] & 0xff000000) >> 3*8), + ((state[2] & 0x00ffffff) << 0*8), + ((state[3] & 0xffffff00) >> 1*8), + ((state[3] & 0x000000ff) << 2*8) | ((state[4] & 0xffff0000) >> 2*8), + ((state[4] & 0x0000ffff) << 1*8), + }; + + for (int i = 0; i < 7; i++){ + uint32_t x = triples[i]; + base64[i*4 + 0] = table[(x >> 3*6) % 64]; + base64[i*4 + 1] = table[(x >> 2*6) % 64]; + base64[i*4 + 2] = table[(x >> 1*6) % 64]; + base64[i*4 + 3] = table[(x >> 0*6) % 64]; + } + + base64[SHA1_BASE64_SIZE - 2] = '='; + if (zero_terminate) base64[SHA1_BASE64_SIZE - 1] = '\0'; + return *this; + } +}; From 05fbb61a31aefa0f44210fc440223a1cafd8c41f Mon Sep 17 00:00:00 2001 From: Oleksandr Yermoshenko Date: Wed, 12 May 2021 15:38:33 +0300 Subject: [PATCH 2/9] Added library feature for fast reusing already compressed files and incremental leanifying. --- Leanify.vcxproj | 2 + Leanify.vcxproj.filters | 6 ++ library.cpp | 128 ++++++++++++++++++++++++++++++++++++++++ library.h | 31 ++++++++++ main.cpp | 53 +++++++++++++++-- 5 files changed, 216 insertions(+), 4 deletions(-) create mode 100644 library.cpp create mode 100644 library.h diff --git a/Leanify.vcxproj b/Leanify.vcxproj index 44dc811..3a71618 100644 --- a/Leanify.vcxproj +++ b/Leanify.vcxproj @@ -319,6 +319,7 @@ + @@ -400,6 +401,7 @@ + diff --git a/Leanify.vcxproj.filters b/Leanify.vcxproj.filters index a10a695..7f3d803 100644 --- a/Leanify.vcxproj.filters +++ b/Leanify.vcxproj.filters @@ -249,6 +249,9 @@ Source Files + + Source Files + @@ -326,6 +329,9 @@ Header Files\formats + + Header Files + diff --git a/library.cpp b/library.cpp new file mode 100644 index 0000000..d71117b --- /dev/null +++ b/library.cpp @@ -0,0 +1,128 @@ +#include "SHA1/sha1.hpp" +#include "library.h" +#include +#include +#include +#include +#include +#include + +// Using this library may require additional compiler/linker options. +// GNU implementation prior to 9.1 requires linking with -lstdc++fs and LLVM implementation +// prior to LLVM 9.0 requires linking with -lc++fs. +#include +#include + +#ifdef _WIN32 +#include +#endif + + +#define PATH_SEPARATOR '/' + +class Storage { + public: + virtual LibraryEntry* GetEntry(void* data, size_t dataSize, const char* tag) = 0; + virtual const std::string& GetStorageName() = 0; +}; + +static Storage* LibraryStorage = NULL; + + + + + +class FileEntry : public LibraryEntry { + private: + std::string _path; + public: + FileEntry(const std::string& path) : _path(path) {} + + public: + virtual bool isExists() { + return std::filesystem::exists(_path); + } + virtual void Save(void* data, size_t dataSize) { + std::ofstream fout(_path, std::ios_base::binary); + fout.write((const char*)data, dataSize); + fout.close(); + } + virtual size_t Load(void* data, size_t dataSize) { + auto size = std::filesystem::file_size(_path); + std::ifstream fin(_path, std::ios_base::binary); + fin.read((char*)data, size); + fin.close(); + return size; + } +}; + +int dirExists(const char* path) { + struct stat info; + + if (stat(path, &info) != 0) + return 0; + else if (info.st_mode & S_IFDIR) + return 1; + else + return 0; +} + +class DirectoryStorage : public Storage { + private: + std::string _pathToDirectory; + protected: + LibraryEntry* CreateEntry(const std::string hash) { + auto firstTwo = std::string(hash, 0, 2); + + auto directory = _pathToDirectory + PATH_SEPARATOR + firstTwo; + std::filesystem::create_directories(directory); + auto fullPath = directory + PATH_SEPARATOR + hash; + return new FileEntry(fullPath); + } + + + public: + DirectoryStorage(const std::string pathToDirectory) + : _pathToDirectory(pathToDirectory) { + if (_pathToDirectory == "*") { + using Codecvt = std::codecvt; + std::wstring_convert converter; + _pathToDirectory = converter.to_bytes(std::filesystem::temp_directory_path()); + _pathToDirectory += "leanify_library"; + } + std::filesystem::create_directories(_pathToDirectory); + if (!dirExists(_pathToDirectory.c_str())) + throw std::exception("Library directory not exists."); + } + + LibraryEntry* GetEntry(void* data, size_t dataSize, const char* tag) { + auto hash = sha1(tag); + hash.add(data, dataSize); + char hex[SHA1_HEX_SIZE]; + hash.print_hex(hex); + return CreateEntry(hex); + } + const std::string& GetStorageName() { + return _pathToDirectory; + } +}; + + +#ifdef _WIN32 +void Library::Initialize(const std::wstring& library) { + char mbs[MAX_PATH] = { 0 }; + WideCharToMultiByte(CP_ACP, 0, library.c_str(), -1, mbs, sizeof(mbs) - 1, nullptr, nullptr); + LibraryStorage = new DirectoryStorage(mbs); +} +#else +void Library::Initialize(const std::string& library) { +} +#endif +LibraryEntry* Library::GetEntry(void* data, size_t dataSize, const char* tag) { + return LibraryStorage ? LibraryStorage->GetEntry(data, dataSize, tag): NULL; +} + +const std::string& Library::GetStorageName() { + static std::string none = "None"; + return LibraryStorage ? LibraryStorage->GetStorageName() : none; +} \ No newline at end of file diff --git a/library.h b/library.h new file mode 100644 index 0000000..d701549 --- /dev/null +++ b/library.h @@ -0,0 +1,31 @@ +#ifndef LIBRARY_H_ +#define LIBRARY_H_ + +#include + +class LibraryEntry; + +class Library +{ + public: +#ifdef _WIN32 + static void Initialize(const std::wstring& library); +#else + static void Initialize(const std::string& library); +#endif + + static LibraryEntry* GetEntry(void* data, size_t dataSize, const char* tag); + static const std::string& GetStorageName(); + +}; + +class LibraryEntry +{ + public: + virtual bool isExists() = 0; + virtual void Save(void* data, size_t dataSize) = 0; + virtual size_t Load(void* data, size_t dataSize) = 0; + virtual ~LibraryEntry() {} +}; + +#endif \ No newline at end of file diff --git a/main.cpp b/main.cpp index 21929ba..1e25e97 100644 --- a/main.cpp +++ b/main.cpp @@ -12,6 +12,7 @@ #include #include "fileio.h" +#include "library.h" #include "formats/jpeg.h" #include "formats/png.h" #include "formats/zip.h" @@ -62,7 +63,22 @@ int ProcessFile(const std::string& file_path) { if (input_file.IsOK()) { size_t original_size = input_file.GetSize(); - size_t new_size = LeanifyFile(input_file.GetFilePionter(), original_size, 0, filename); + size_t new_size(0); + string libraryTag = ToString(iterations); + + bool reusedFromLibrary = false; + + auto libraryEntry = Library::GetEntry(input_file.GetFilePionter(), original_size, libraryTag.c_str()); + if (!libraryEntry || !libraryEntry->isExists()) { + new_size = LeanifyFile(input_file.GetFilePionter(), original_size, 0, filename); + if (libraryEntry) + libraryEntry->Save(input_file.GetFilePionter(), new_size); + delete libraryEntry; + } else if (libraryEntry->isExists()) { + new_size = libraryEntry->Load(input_file.GetFilePionter(), original_size); + delete libraryEntry; + reusedFromLibrary = true; + } std::string log; if (parallel_processing) @@ -72,7 +88,7 @@ int ProcessFile(const std::string& file_path) { BuildSize(original_size) + " -> " + BuildSize(new_size) + - "\tLeanified: " + + ((!reusedFromLibrary) ? "\tLeanified: " : "\tReused: ") + BuildSize(original_size - new_size) + " (" + ToString(100 - 100.0 * new_size / original_size) + @@ -109,6 +125,8 @@ void PrintInfo() { " -q, --quiet No output to stdout.\n" " -v, --verbose Verbose output.\n" " -p, --parallel Distribute all tasks to all CPUs.\n" + " -l, --library Use library to store and reuse alreary compressed files." + " Set * will automatically use temporary folder." " --keep-exif Do not remove Exif.\n" " --keep-icc Do not remove ICC profile.\n" "\n" @@ -147,7 +165,8 @@ int EnqueueProcessFileTask(const char* file_path, const struct stat* sb = nullpt #ifdef _WIN32 int main() { int argc; - wchar_t** argv = CommandLineToArgvW(GetCommandLineW(), &argc); + wchar_t* command_line = GetCommandLineW(); + wchar_t** argv = CommandLineToArgvW(command_line, &argc); #else int main(int argc, char** argv) { #endif // _WIN32 @@ -157,7 +176,11 @@ int main(int argc, char** argv) { iterations = 15; depth = 1; max_depth = INT_MAX; - +#ifdef _WIN32 + std::wstring library_path; +#else + std::string library_path; +#endif // _WIN32 #ifdef _WIN32 is_pause = !getenv("PROMPT"); #endif // _WIN32 @@ -196,6 +219,12 @@ int main(int argc, char** argv) { } } break; + case 'l': + if (i < argc - 1) { + auto value = argv[i + ++num_optargs]; + library_path = value; + } + break; case 'q': cout.setstate(std::ios::failbit); is_verbose = false; @@ -226,6 +255,9 @@ int main(int argc, char** argv) { } else if (STRCMP(argv[i] + j + 1, "parallel") == 0) { j += 7; argv[i][j + 1] = 'p'; + } else if (STRCMP(argv[i] + j + 1, "library") == 0) { + j += 6; + argv[i][j + 1] = 'l'; } else if (STRCMP(argv[i] + j + 1, "keep-exif") == 0) { j += 9; Jpeg::keep_exif_ = true; @@ -278,6 +310,19 @@ int main(int argc, char** argv) { cout << std::fixed; cout.precision(2); + if (library_path.size() > 0) { + try { + Library::Initialize(library_path); + cout << "Library storage: " << Library::GetStorageName() << endl << endl; + } + catch(std::exception e) { + cerr << "Error: " << e.what() << endl << endl; + PrintInfo(); + return 1; + } + } + + // support multiple input file do { TraversePath(argv[i], EnqueueProcessFileTask); From 4b7854c9974c0ae82e222298ae961bdfd101291c Mon Sep 17 00:00:00 2001 From: Alex A Yermoshenko Date: Wed, 12 May 2021 23:20:06 +0300 Subject: [PATCH 3/9] Compiled using make file --- Makefile | 2 +- library.cpp | 11 +++++++++-- main.cpp | 2 +- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 12da93b..62d984e 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -LEANIFY_SRC := leanify.cpp main.cpp utils.cpp $(wildcard formats/*.cpp) +LEANIFY_SRC := leanify.cpp main.cpp utils.cpp library.cpp $(wildcard formats/*.cpp) LZMA_OBJ := lib/LZMA/Alloc.o lib/LZMA/LzFind.o lib/LZMA/LzmaDec.o lib/LZMA/LzmaEnc.o MOZJPEG_OBJ := lib/mozjpeg/jaricom.o lib/mozjpeg/jcapimin.o lib/mozjpeg/jcarith.o lib/mozjpeg/jcext.o lib/mozjpeg/jchuff.o lib/mozjpeg/jcmarker.o lib/mozjpeg/jcmaster.o lib/mozjpeg/jcomapi.o lib/mozjpeg/jcparam.o lib/mozjpeg/jcphuff.o lib/mozjpeg/jctrans.o lib/mozjpeg/jdapimin.o lib/mozjpeg/jdarith.o lib/mozjpeg/jdatadst.o lib/mozjpeg/jdatasrc.o lib/mozjpeg/jdcoefct.o lib/mozjpeg/jdhuff.o lib/mozjpeg/jdinput.o lib/mozjpeg/jdmarker.o lib/mozjpeg/jdphuff.o lib/mozjpeg/jdtrans.o lib/mozjpeg/jerror.o lib/mozjpeg/jmemmgr.o lib/mozjpeg/jmemnobs.o lib/mozjpeg/jsimd_none.o lib/mozjpeg/jutils.o PUGIXML_OBJ := lib/pugixml/pugixml.o diff --git a/library.cpp b/library.cpp index d71117b..a3d4382 100644 --- a/library.cpp +++ b/library.cpp @@ -67,6 +67,13 @@ int dirExists(const char* path) { return 0; } +template +struct codecvt : std::codecvt +{ + ~codecvt() + { } +}; + class DirectoryStorage : public Storage { private: std::string _pathToDirectory; @@ -85,14 +92,14 @@ class DirectoryStorage : public Storage { DirectoryStorage(const std::string pathToDirectory) : _pathToDirectory(pathToDirectory) { if (_pathToDirectory == "*") { - using Codecvt = std::codecvt; + using Codecvt = codecvt; std::wstring_convert converter; _pathToDirectory = converter.to_bytes(std::filesystem::temp_directory_path()); _pathToDirectory += "leanify_library"; } std::filesystem::create_directories(_pathToDirectory); if (!dirExists(_pathToDirectory.c_str())) - throw std::exception("Library directory not exists."); + throw std::runtime_error("Library directory not exists."); } LibraryEntry* GetEntry(void* data, size_t dataSize, const char* tag) { diff --git a/main.cpp b/main.cpp index 1e25e97..fa40fba 100644 --- a/main.cpp +++ b/main.cpp @@ -315,7 +315,7 @@ int main(int argc, char** argv) { Library::Initialize(library_path); cout << "Library storage: " << Library::GetStorageName() << endl << endl; } - catch(std::exception e) { + catch(std::runtime_error e) { cerr << "Error: " << e.what() << endl << endl; PrintInfo(); return 1; From 39f529ae8204268ebdb1f246e26675dfd924f343 Mon Sep 17 00:00:00 2001 From: Oleksandr Yermoshenko Date: Thu, 13 May 2021 15:26:36 +0300 Subject: [PATCH 4/9] Added compilation flag '-lstdc++fs' and linker flag '-lc++fs' --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 5a5424f..d78902d 100644 --- a/Makefile +++ b/Makefile @@ -7,8 +7,8 @@ ZOPFLIPNG_OBJ := lib/zopflipng/lodepng/lodepng.o lib/zopflipng/lodepng/lodepng CFLAGS += -Wall -Wextra -Wno-unused-parameter -Werror -O3 -msse2 -mfpmath=sse -flto CPPFLAGS += -I./lib -CXXFLAGS += $(CFLAGS) -std=c++17 -fno-rtti -LDFLAGS += -flto -lpthread +CXXFLAGS += $(CFLAGS) -std=c++17 -fno-rtti -lstdc++fs +LDFLAGS += -flto -lpthread -lc++fs ifeq ($(OS), Windows_NT) SYSTEM := Windows From 9eb17cf8ba35b4e0fd1204807a5ca5f02682abb3 Mon Sep 17 00:00:00 2001 From: Oleksandr Yermoshenko Date: Thu, 13 May 2021 15:27:34 +0300 Subject: [PATCH 5/9] Library code cleanup, and initializing witn non unicode path. --- library.cpp | 86 ++++++++++++++++++++++++++--------------------------- main.cpp | 3 +- 2 files changed, 44 insertions(+), 45 deletions(-) diff --git a/library.cpp b/library.cpp index a3d4382..8057814 100644 --- a/library.cpp +++ b/library.cpp @@ -1,14 +1,15 @@ -#include "SHA1/sha1.hpp" #include "library.h" -#include -#include + #include #include #include #include -// Using this library may require additional compiler/linker options. -// GNU implementation prior to 9.1 requires linking with -lstdc++fs and LLVM implementation +#include +#include + +// Using this library may require additional compiler/linker options. +// GNU implementation prior to 9.1 requires linking with -lstdc++fs and LLVM implementation // prior to LLVM 9.0 requires linking with -lc++fs. #include #include @@ -17,8 +18,9 @@ #include #endif +#include -#define PATH_SEPARATOR '/' +constexpr auto PATH_SEPARATOR = '/'; class Storage { public: @@ -28,32 +30,29 @@ class Storage { static Storage* LibraryStorage = NULL; - - - - class FileEntry : public LibraryEntry { private: std::string _path; - public: - FileEntry(const std::string& path) : _path(path) {} - - public: - virtual bool isExists() { - return std::filesystem::exists(_path); - } - virtual void Save(void* data, size_t dataSize) { - std::ofstream fout(_path, std::ios_base::binary); - fout.write((const char*)data, dataSize); - fout.close(); - } - virtual size_t Load(void* data, size_t dataSize) { - auto size = std::filesystem::file_size(_path); - std::ifstream fin(_path, std::ios_base::binary); - fin.read((char*)data, size); - fin.close(); - return size; - } + + public: + FileEntry(const std::string& path) : _path(path) {} + + public: + virtual bool isExists() { + return std::filesystem::exists(_path); + } + virtual void Save(void* data, size_t dataSize) { + std::ofstream fout(_path, std::ios_base::binary); + fout.write((const char*)data, dataSize); + fout.close(); + } + virtual size_t Load(void* data, size_t dataSize) { + auto size = std::filesystem::file_size(_path); + std::ifstream fin(_path, std::ios_base::binary); + fin.read((char*)data, size); + fin.close(); + return size; + } }; int dirExists(const char* path) { @@ -67,16 +66,21 @@ int dirExists(const char* path) { return 0; } -template -struct codecvt : std::codecvt -{ - ~codecvt() - { } +template +struct codecvt : std::codecvt { + ~codecvt() {} }; +std::string pathToString(const std::filesystem::path& path) { + typedef codecvt Codecvt; + std::wstring_convert converter; + return converter.to_bytes(std::filesystem::temp_directory_path()); +} + class DirectoryStorage : public Storage { private: std::string _pathToDirectory; + protected: LibraryEntry* CreateEntry(const std::string hash) { auto firstTwo = std::string(hash, 0, 2); @@ -87,19 +91,15 @@ class DirectoryStorage : public Storage { return new FileEntry(fullPath); } - public: - DirectoryStorage(const std::string pathToDirectory) - : _pathToDirectory(pathToDirectory) { + DirectoryStorage(const std::string pathToDirectory) : _pathToDirectory(pathToDirectory) { if (_pathToDirectory == "*") { - using Codecvt = codecvt; - std::wstring_convert converter; - _pathToDirectory = converter.to_bytes(std::filesystem::temp_directory_path()); + _pathToDirectory = pathToString(std::filesystem::temp_directory_path()); _pathToDirectory += "leanify_library"; } std::filesystem::create_directories(_pathToDirectory); if (!dirExists(_pathToDirectory.c_str())) - throw std::runtime_error("Library directory not exists."); + throw std::runtime_error("Library directory not exists: " + _pathToDirectory); } LibraryEntry* GetEntry(void* data, size_t dataSize, const char* tag) { @@ -114,7 +114,6 @@ class DirectoryStorage : public Storage { } }; - #ifdef _WIN32 void Library::Initialize(const std::wstring& library) { char mbs[MAX_PATH] = { 0 }; @@ -123,10 +122,11 @@ void Library::Initialize(const std::wstring& library) { } #else void Library::Initialize(const std::string& library) { + LibraryStorage = new DirectoryStorage(library); } #endif LibraryEntry* Library::GetEntry(void* data, size_t dataSize, const char* tag) { - return LibraryStorage ? LibraryStorage->GetEntry(data, dataSize, tag): NULL; + return LibraryStorage ? LibraryStorage->GetEntry(data, dataSize, tag) : NULL; } const std::string& Library::GetStorageName() { diff --git a/main.cpp b/main.cpp index 0ba0adf..f3c4e26 100644 --- a/main.cpp +++ b/main.cpp @@ -314,8 +314,7 @@ int main(int argc, char** argv) { try { Library::Initialize(library_path); cout << "Library storage: " << Library::GetStorageName() << endl << endl; - } - catch(std::runtime_error e) { + } catch (const std::runtime_error& e) { cerr << "Error: " << e.what() << endl << endl; PrintInfo(); return 1; From fafb05eaf34da5944ff192bcd2cfed9790246f54 Mon Sep 17 00:00:00 2001 From: Oleksandr Yermoshenko Date: Thu, 13 May 2021 16:34:59 +0300 Subject: [PATCH 6/9] Added wrapped filesystem to fix gcc compillation --- Leanify.vcxproj | 1 + Leanify.vcxproj.filters | 3 ++ Makefile | 4 +-- filesystem.h | 71 +++++++++++++++++++++++++++++++++++++++++ library.cpp | 11 +++---- 5 files changed, 81 insertions(+), 9 deletions(-) create mode 100644 filesystem.h diff --git a/Leanify.vcxproj b/Leanify.vcxproj index 39157c4..e9eba4b 100644 --- a/Leanify.vcxproj +++ b/Leanify.vcxproj @@ -384,6 +384,7 @@ + diff --git a/Leanify.vcxproj.filters b/Leanify.vcxproj.filters index 7f3d803..12aa148 100644 --- a/Leanify.vcxproj.filters +++ b/Leanify.vcxproj.filters @@ -332,6 +332,9 @@ Header Files + + Header Files + diff --git a/Makefile b/Makefile index d78902d..5a5424f 100644 --- a/Makefile +++ b/Makefile @@ -7,8 +7,8 @@ ZOPFLIPNG_OBJ := lib/zopflipng/lodepng/lodepng.o lib/zopflipng/lodepng/lodepng CFLAGS += -Wall -Wextra -Wno-unused-parameter -Werror -O3 -msse2 -mfpmath=sse -flto CPPFLAGS += -I./lib -CXXFLAGS += $(CFLAGS) -std=c++17 -fno-rtti -lstdc++fs -LDFLAGS += -flto -lpthread -lc++fs +CXXFLAGS += $(CFLAGS) -std=c++17 -fno-rtti +LDFLAGS += -flto -lpthread ifeq ($(OS), Windows_NT) SYSTEM := Windows diff --git a/filesystem.h b/filesystem.h new file mode 100644 index 0000000..356ee26 --- /dev/null +++ b/filesystem.h @@ -0,0 +1,71 @@ +// https://stackoverflow.com/questions/53365538/how-to-determine-whether-to-use-filesystem-or-experimental-filesystem + +// We haven't checked which filesystem to include yet +#ifndef INCLUDE_STD_FILESYSTEM_EXPERIMENTAL + +// Check for feature test macro for +#if defined(__cpp_lib_filesystem) +#define INCLUDE_STD_FILESYSTEM_EXPERIMENTAL 0 + +// Check for feature test macro for +#elif defined(__cpp_lib_experimental_filesystem) +#define INCLUDE_STD_FILESYSTEM_EXPERIMENTAL 1 + +// We can't check if headers exist... +// Let's assume experimental to be safe +#elif !defined(__has_include) +#define INCLUDE_STD_FILESYSTEM_EXPERIMENTAL 1 + +// Check if the header "" exists +#elif __has_include() + +// If we're compiling on Visual Studio and are not compiling with C++17, we need to use experimental +#ifdef _MSC_VER + +// Check and include header that defines "_HAS_CXX17" +#if __has_include() +#include + +// Check for enabled C++17 support +#if defined(_HAS_CXX17) && _HAS_CXX17 +// We're using C++17, so let's use the normal version +#define INCLUDE_STD_FILESYSTEM_EXPERIMENTAL 0 +#endif +#endif + +// If the marco isn't defined yet, that means any of the other VS specific checks failed, so we need to use experimental +#ifndef INCLUDE_STD_FILESYSTEM_EXPERIMENTAL +#define INCLUDE_STD_FILESYSTEM_EXPERIMENTAL 1 +#endif + +// Not on Visual Studio. Let's use the normal version +#else // #ifdef _MSC_VER +#define INCLUDE_STD_FILESYSTEM_EXPERIMENTAL 0 +#endif + +// Check if the header "" exists +#elif __has_include() +#define INCLUDE_STD_FILESYSTEM_EXPERIMENTAL 1 + +// Fail if neither header is available with a nice error message +#else +#error Could not find system header "" or "" +#endif + +// We priously determined that we need the exprimental version +#if INCLUDE_STD_FILESYSTEM_EXPERIMENTAL +// Include it +#include + +// We need the alias from std::experimental::filesystem to std::filesystem +namespace std { +namespace filesystem = experimental::filesystem; +} + +// We have a decent compiler and can use the normal version +#else +// Include it +#include +#endif + +#endif // #ifndef INCLUDE_STD_FILESYSTEM_EXPERIMENTAL#pragma once diff --git a/library.cpp b/library.cpp index 8057814..d3fcf3b 100644 --- a/library.cpp +++ b/library.cpp @@ -8,10 +8,6 @@ #include #include -// Using this library may require additional compiler/linker options. -// GNU implementation prior to 9.1 requires linking with -lstdc++fs and LLVM implementation -// prior to LLVM 9.0 requires linking with -lc++fs. -#include #include #ifdef _WIN32 @@ -19,6 +15,7 @@ #endif #include +#include "filesystem.h" constexpr auto PATH_SEPARATOR = '/'; @@ -55,10 +52,10 @@ class FileEntry : public LibraryEntry { } }; -int dirExists(const char* path) { +int is_directory_exists(const std::string& path) { struct stat info; - if (stat(path, &info) != 0) + if (stat(path.c_str(), &info) != 0) return 0; else if (info.st_mode & S_IFDIR) return 1; @@ -98,7 +95,7 @@ class DirectoryStorage : public Storage { _pathToDirectory += "leanify_library"; } std::filesystem::create_directories(_pathToDirectory); - if (!dirExists(_pathToDirectory.c_str())) + if (!is_directory_exists(_pathToDirectory.c_str())) throw std::runtime_error("Library directory not exists: " + _pathToDirectory); } From 9fc15811275e55b86c34cad01c55f61d47402aa0 Mon Sep 17 00:00:00 2001 From: Oleksandr Yermoshenko Date: Thu, 13 May 2021 16:50:35 +0300 Subject: [PATCH 7/9] Added linker flag '-lstdc++fs' --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 5a5424f..c71abe4 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ ZOPFLIPNG_OBJ := lib/zopflipng/lodepng/lodepng.o lib/zopflipng/lodepng/lodepng CFLAGS += -Wall -Wextra -Wno-unused-parameter -Werror -O3 -msse2 -mfpmath=sse -flto CPPFLAGS += -I./lib CXXFLAGS += $(CFLAGS) -std=c++17 -fno-rtti -LDFLAGS += -flto -lpthread +LDFLAGS += -flto -lpthread -lstdc++fs ifeq ($(OS), Windows_NT) SYSTEM := Windows From 17e0f333019b1d71f17a8b928b6a9b4bcbf80909 Mon Sep 17 00:00:00 2001 From: Oleksandr Yermoshenko Date: Thu, 13 May 2021 16:58:00 +0300 Subject: [PATCH 8/9] Fixed Mac compillation, -lstdc++fs flag adds only on Linux --- Makefile | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index c71abe4..d992651 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ ZOPFLIPNG_OBJ := lib/zopflipng/lodepng/lodepng.o lib/zopflipng/lodepng/lodepng CFLAGS += -Wall -Wextra -Wno-unused-parameter -Werror -O3 -msse2 -mfpmath=sse -flto CPPFLAGS += -I./lib CXXFLAGS += $(CFLAGS) -std=c++17 -fno-rtti -LDFLAGS += -flto -lpthread -lstdc++fs +LDFLAGS += -flto -lpthread ifeq ($(OS), Windows_NT) SYSTEM := Windows @@ -22,6 +22,11 @@ ifeq ($(SYSTEM), Linux) LDFLAGS += -fuse-ld=gold endif +# -lstdc++fs supported only on Linux +ifeq ($(SYSTEM), Linux) + LDFLAGS += -lstdc++fs +endif + ifeq ($(SYSTEM), Darwin) LDLIBS += -liconv else From 5f18ef7712330e649dbe9218b6210a175380b64a Mon Sep 17 00:00:00 2001 From: "Alex A. Yermoshenko" Date: Fri, 14 Oct 2022 14:57:22 +0300 Subject: [PATCH 9/9] Fixed library hashes: use more parameners when building salt. --- main.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/main.cpp b/main.cpp index e6f45ba..58e8281 100644 --- a/main.cpp +++ b/main.cpp @@ -64,7 +64,8 @@ int ProcessFile(const std::string& file_path) { size_t original_size = input_file.GetSize(); size_t new_size(0); - string libraryTag = ToString(iterations); + string libraryTag = ToString(iterations) + (zopflipng_lossy_transparent ? "lossy" : "lossless") + + (is_fast ? "no_zopflipng" : "zopflipng"); bool reusedFromLibrary = false; auto filePointer = input_file.GetFilePointer();