Add .zst archive file support, and zstd compression to .zip

Author: warmenhoven

Makefile.common

@@ -2022,7 +2022,8 @@ ifeq ($(HAVE_ZSTD),1)
            $(DEPS_DIR)/zstd/lib/decompress/zstd_decompress.o \
            $(DEPS_DIR)/zstd/lib/decompress/zstd_decompress_block.o
 
-   OBJ +=  $(ZSOBJ)
+   OBJ +=  $(LIBRETRO_COMM_DIR)/file/archive_file_zstd.o \
+           $(ZSOBJ)
 endif
 
 ifeq ($(HAVE_IBXM), 1)

deps/7zip/7zDec.c

@@ -20,6 +20,10 @@
 #include "Ppmd7.h"
 #endif
 
+#ifdef HAVE_ZSTD
+#include <zstd.h>
+#endif
+
 #define k_Copy 0
 #define k_Delta 3
 #define k_LZMA2 0x21
@@ -32,6 +36,10 @@
 #define k_ARMT  0x3030701
 #define k_SPARC 0x3030805
 
+#ifdef HAVE_ZSTD
+#define k_ZSTD 0x4F71101
+#endif
+
 
 #ifdef _7ZIP_PPMD_SUPPPORT
 
@@ -280,6 +288,9 @@ static BoolInt IS_MAIN_METHOD(uint32_t m)
     #endif
     #ifdef _7ZIP_PPMD_SUPPPORT
     case k_PPMD:
+    #endif
+    #ifdef HAVE_ZSTD
+    case k_ZSTD:
     #endif
       return True;
   }
@@ -446,6 +457,20 @@ static SRes SzFolder_Decode2(const CSzFolder *folder,
         RINOK(SzDecodePpmd(propsData + coder->PropsOffset, coder->PropsSize, inSize, inStream, outBufCur, outSizeCur, allocMain));
       }
       #endif
+      #ifdef HAVE_ZSTD
+      else if (coder->MethodID == k_ZSTD)
+      {
+        Byte *inBuf = (Byte *)ISzAlloc_Alloc(allocMain, (size_t)inSize);
+        size_t ret;
+        if (!inBuf)
+          return SZ_ERROR_MEM;
+        RINOK(SzDecodeCopy(inSize, inStream, inBuf));
+        ret = ZSTD_decompress(outBufCur, outSizeCur, inBuf, (size_t)inSize);
+        ISzAlloc_Free(allocMain, inBuf);
+        if (ZSTD_isError(ret) || ret != outSizeCur)
+          return SZ_ERROR_DATA;
+      }
+      #endif
       else
         return SZ_ERROR_UNSUPPORTED;
     }

griffin/griffin.c

@@ -134,6 +134,10 @@ ARCHIVE FILE
 #include "../libretro-common/file/archive_file_7z.c"
 #endif
 
+#ifdef HAVE_ZSTD
+#include "../libretro-common/file/archive_file_zstd.c"
+#endif
+
 /*============================================================
 COMPRESSION
 ============================================================ */

libretro-common/file/archive_file.c

@@ -581,9 +581,18 @@ const struct file_archive_file_backend *file_archive_get_7z_file_backend(void)
 #endif
 }
 
+const struct file_archive_file_backend *file_archive_get_zstd_file_backend(void)
+{
+#ifdef HAVE_ZSTD
+   return &zstd_backend;
+#else
+   return NULL;
+#endif
+}
+
 const struct file_archive_file_backend* file_archive_get_file_backend(const char *path)
 {
-#if defined(HAVE_7ZIP) || defined(HAVE_ZLIB)
+#if defined(HAVE_7ZIP) || defined(HAVE_ZLIB) || defined(HAVE_ZSTD)
    char newpath[PATH_MAX_LENGTH];
    const char *file_ext          = NULL;
    char *last                    = NULL;
@@ -606,6 +615,11 @@ const struct file_archive_file_backend* file_archive_get_file_backend(const char
       )
       return &zlib_backend;
 #endif
+
+#ifdef HAVE_ZSTD
+   if (string_is_equal_noncase(file_ext, "zst"))
+      return &zstd_backend;
+#endif
 #endif
 
    return NULL;

libretro-common/file/archive_file_zlib.c

@@ -31,6 +31,10 @@
 
 #include <zlib.h>
 
+#ifdef HAVE_ZSTD
+#include <zstd.h>
+#endif
+
 #ifndef CENTRAL_FILE_HEADER_SIGNATURE
 #define CENTRAL_FILE_HEADER_SIGNATURE 0x02014b50
 #endif
@@ -44,7 +48,8 @@
 enum file_archive_compression_mode
 {
    ZIP_MODE_STORED   = 0,
-   ZIP_MODE_DEFLATED = 8
+   ZIP_MODE_DEFLATED = 8,
+   ZIP_MODE_ZSTD     = 93
 };
 
 typedef struct
@@ -163,6 +168,19 @@ static bool zlib_stream_decompress_data_to_file_init(
          return false;
       }
    }
+#ifdef HAVE_ZSTD
+   else if (cmode == ZIP_MODE_ZSTD)
+   {
+      /* Allocate a buffer to read compressed data into;
+       * decompression is done in one shot during iterate */
+      zip_context->tmpbuf = (uint8_t*)malloc(csize);
+      if (!zip_context->tmpbuf)
+      {
+         zip_context_free_stream(zip_context, false);
+         return false;
+      }
+   }
+#endif
 
    return true;
 }
@@ -243,6 +261,45 @@ static int zlib_stream_decompress_data_to_file_iterate(
 
       return 0;   /* still more data to process */
    }
+#ifdef HAVE_ZSTD
+   else if (zip_context->cmode == ZIP_MODE_ZSTD)
+   {
+      size_t result;
+
+#ifdef HAVE_MMAP
+      if (state->archive_mmap_data)
+      {
+         result = ZSTD_decompress(
+               zip_context->decompressed_data, zip_context->usize,
+               state->archive_mmap_data + (size_t)zip_context->fdoffset,
+               zip_context->csize);
+      }
+      else
+#endif
+      {
+         /* Read all compressed data, then decompress */
+         filestream_seek(state->archive_file,
+               zip_context->fdoffset,
+               RETRO_VFS_SEEK_POSITION_START);
+         if (filestream_read(state->archive_file,
+                  zip_context->tmpbuf, zip_context->csize) < 0)
+            return -1;
+
+         result = ZSTD_decompress(
+               zip_context->decompressed_data, zip_context->usize,
+               zip_context->tmpbuf, zip_context->csize);
+      }
+
+      if (ZSTD_isError(result))
+         return -1;
+
+      free(zip_context->tmpbuf);
+      zip_context->tmpbuf = NULL;
+
+      handle->data = zip_context->decompressed_data;
+      return 1;
+   }
+#endif
 
    /* No idea what kind of compression this is */
    return -1;