Update to LZMA SDK 25.01 and always enable multithreading.

Author: fancycode

.gitignore

@@ -4,6 +4,7 @@ dist
 *.pyc
 *.egg-info
 *.so
+*.patch~
 *.pyd
 RELEASE-VERSION
 /.pc

patches/series

@@ -1,2 +1 @@
 streaming_encoder.patch
-strict_prototype.patch

patches/streaming_encoder.patch

@@ -2,47 +2,51 @@ Index: pylzma/src/sdk/C/LzmaEnc.c
 ===================================================================
 --- pylzma.orig/src/sdk/C/LzmaEnc.c
 +++ pylzma/src/sdk/C/LzmaEnc.c
-@@ -2259,8 +2259,9 @@ void LzmaEnc_Destroy(CLzmaEncHandle p, I
+@@ -2385,9 +2385,9 @@ void LzmaEnc_Destroy(CLzmaEncHandle p, I
  }
 
 
+-Z7_NO_INLINE
 -static SRes LzmaEnc_CodeOneBlock(CLzmaEnc *p, UInt32 maxPackSize, UInt32 maxUnpackSize)
 +SRes LzmaEnc_CodeOneBlock(CLzmaEncHandle pp, UInt32 maxPackSize, UInt32 maxUnpackSize)
  {
 +  CLzmaEnc *p = (CLzmaEnc *) pp;
    UInt32 nowPos32, startPos32;
    if (p->needInit)
    {
-@@ -2715,7 +2716,7 @@ static SRes LzmaEnc_AllocAndInit(CLzmaEn
+@@ -2872,7 +2872,7 @@ static SRes LzmaEnc_AllocAndInit(CLzmaEn
    return SZ_OK;
  }
 
--static SRes LzmaEnc_Prepare(CLzmaEncHandle pp, ISeqOutStream *outStream, ISeqInStream *inStream,
-+SRes LzmaEnc_Prepare(CLzmaEncHandle pp, ISeqOutStream *outStream, ISeqInStream *inStream,
+-static SRes LzmaEnc_Prepare(CLzmaEncHandle p,
++SRes LzmaEnc_Prepare(CLzmaEncHandle p,
+     ISeqOutStreamPtr outStream,
+     ISeqInStreamPtr inStream,
      ISzAllocPtr alloc, ISzAllocPtr allocBig)
- {
-   CLzmaEnc *p = (CLzmaEnc *)pp;
-@@ -2974,3 +2975,9 @@ SRes LzmaEncode(Byte *dest, SizeT *destL
-   LzmaEnc_Destroy(p, alloc, allocBig);
-   return res;
+@@ -2883,6 +2883,12 @@ static SRes LzmaEnc_Prepare(CLzmaEncHand
+   return LzmaEnc_AllocAndInit(p, 0, alloc, allocBig);
  }
-+
+ 
 +BoolInt LzmaEnc_IsFinished(CLzmaEncHandle pp)
 +{
 +  CLzmaEnc *p = (CLzmaEnc *)pp;
 +  return p->finished;
 +}
++
+ SRes LzmaEnc_PrepareForLzma2(CLzmaEncHandle p,
+     ISeqInStreamPtr inStream, UInt32 keepWindowSize,
+     ISzAllocPtr alloc, ISzAllocPtr allocBig)
 Index: pylzma/src/sdk/C/LzmaEnc.h
 ===================================================================
 --- pylzma.orig/src/sdk/C/LzmaEnc.h
 +++ pylzma/src/sdk/C/LzmaEnc.h
-@@ -73,4 +73,11 @@ SRes LzmaEncode(Byte *dest, SizeT *destL
+@@ -82,4 +82,11 @@ SRes LzmaEncode(Byte *dest, SizeT *destL
 
  EXTERN_C_END
 
 +/* ---------- Streaming Interface ---------- */
 +
-+SRes LzmaEnc_Prepare(CLzmaEncHandle pp, ISeqOutStream *outStream, ISeqInStream *inStream, ISzAllocPtr alloc, ISzAllocPtr allocBig);
++SRes LzmaEnc_Prepare(CLzmaEncHandle pp, ISeqOutStreamPtr outStream, ISeqInStreamPtr inStream, ISzAllocPtr alloc, ISzAllocPtr allocBig);
 +SRes LzmaEnc_CodeOneBlock(CLzmaEncHandle pp, UInt32 maxPackSize, UInt32 maxUnpackSize);
 +BoolInt LzmaEnc_IsFinished(CLzmaEncHandle pp);
 +void LzmaEnc_Finish(CLzmaEncHandle pp);

patches/strict_prototype.patch

@@ -85,11 +85,6 @@ def __new__(cls, s):
 
 library_dirs = []
 
-# platforms that multithreaded compression is supported on
-mt_platforms = (
-    'win32',
-)
-
 if IS_WINDOWS:
     # don't try to import MSVC compiler on non-windows platforms
     # as this triggers unnecessary warnings
@@ -107,14 +102,7 @@ class MSVCCompiler(object):
 class build_ext(_build_ext):
 
     def build_extension(self, ext):
-        self.with_mt = ENABLE_MULTITHREADING
-        if self.with_mt and not sys.platform in mt_platforms:
-            warn("""\
-Multithreading is not supported on the platform "%s",
-please contact [email protected] for more informations.""" % (sys.platform), UnsupportedPlatformWarning)
-            self.with_mt = False
-
-        if self.with_mt:
+        if ENABLE_MULTITHREADING:
             log.info('adding support for multithreaded compression')
             ext.define_macros.append(('COMPRESS_MF_MT', 1))
             ext.sources += (
@@ -124,7 +112,7 @@ def build_extension(self, ext):
                 'src/sdk/C/Threads.c',
             )
         else:
-            ext.define_macros.append(('_7ZIP_ST', 1))
+            ext.define_macros.append(('Z7_ST', 1))
 
         if isinstance(self.compiler, MSVCCompiler) or getattr(self.compiler, 'compiler_type', '') == 'msvc':
             # set flags only available when using MSVC
@@ -163,6 +151,7 @@ def build_extension(self, ext):
     ('PY_SSIZE_T_CLEAN', 1),
 ]
 lzma_files = (
+    'src/sdk/C/7zStream.c',
     'src/sdk/C/Aes.c',
     'src/sdk/C/AesOpt.c',
     'src/sdk/C/Bcj2.c',
@@ -172,11 +161,14 @@ def build_extension(self, ext):
     'src/sdk/C/CpuArch.c',
     'src/sdk/C/Delta.c',
     'src/sdk/C/LzFind.c',
+    'src/sdk/C/LzFindOpt.c',
     'src/sdk/C/LzmaDec.c',
     'src/sdk/C/LzmaEnc.c',
     'src/sdk/C/Lzma2Dec.c',
     'src/sdk/C/Lzma2Enc.c',
     'src/sdk/C/Sha256.c',
+    'src/sdk/C/Sha256Opt.c',
+    'src/sdk/C/SwapBytes.c',
     'src/sdk/C/Ppmd7.c',
     'src/sdk/C/Ppmd7Dec.c',
 )

setup.py

@@ -130,7 +130,7 @@ pylzma_calculate_key(PyObject *self, PyObject *args, PyObject *kwargs)
 
 const char
 doc_bcj_x86_convert[] = \
-    "bcj_x86_convert(data) -- Perform BCJ x86 conversion.";
+    "bcj_x86_convert(data, [encoding]) -- Perform BCJ x86 conversion.";
 
 static PyObject *
 pylzma_bcj_x86_convert(PyObject *self, PyObject *args)
@@ -150,10 +150,13 @@ pylzma_bcj_x86_convert(PyObject *self, PyObject *args)
 
     result = PyBytes_FromStringAndSize(data, length);
     if (result != NULL) {
-        UInt32 state;
+        UInt32 state = Z7_BRANCH_CONV_ST_X86_STATE_INIT_VAL;
         Py_BEGIN_ALLOW_THREADS
-        x86_Convert_Init(state);
-        x86_Convert((Byte *) PyBytes_AS_STRING(result), length, 0, &state, encoding);
+        if (encoding) {
+            Z7_BRANCH_CONV_ST_ENC(X86)((Byte *) PyBytes_AS_STRING(result), length, 0, &state);
+        } else {
+            Z7_BRANCH_CONV_ST_DEC(X86)((Byte *) PyBytes_AS_STRING(result), length, 0, &state);
+        }
         Py_END_ALLOW_THREADS
     }
 
@@ -163,7 +166,7 @@ pylzma_bcj_x86_convert(PyObject *self, PyObject *args)
 #define DEFINE_BCJ_CONVERTER(id, name) \
 const char \
 doc_bcj_##id##_convert[] = \
-    "bcj_" #id "_convert(data) -- Perform BCJ " #name " conversion."; \
+    "bcj_" #id "_convert(data, [encoding]) -- Perform BCJ " #name " conversion."; \
 \
 static PyObject * \
 pylzma_bcj_##id##_convert(PyObject *self, PyObject *args) \
@@ -184,7 +187,11 @@ pylzma_bcj_##id##_convert(PyObject *self, PyObject *args) \
     result = PyBytes_FromStringAndSize(data, length); \
     if (result != NULL) { \
         Py_BEGIN_ALLOW_THREADS \
-        name##_Convert((Byte *) PyBytes_AS_STRING(result), length, 0, encoding); \
+        if (encoding) { \
+            Z7_BRANCH_CONV_ENC(name)((Byte *) PyBytes_AS_STRING(result), length, 0); \
+        } else { \
+            Z7_BRANCH_CONV_DEC(name)((Byte *) PyBytes_AS_STRING(result), length, 0); \
+        } \
         Py_END_ALLOW_THREADS \
     } \
      \
@@ -259,7 +266,7 @@ pylzma_bcj2_decode(PyObject *self, PyObject *args)
     }
 
     // Conversion must be finished and the output buffer filled completely.
-    if (!Bcj2Dec_IsFinished(&dec)) {
+    if (!Bcj2Dec_IsMaybeFinished(&dec)) {
         goto error;
     } else if (dec.dest != dec.destLim || dec.state != BCJ2_STREAM_MAIN) {
         goto error;
@@ -370,7 +377,7 @@ typedef struct
 
 static Byte
 ReadByte(const IByteIn *pp) {
-    CByteInToLook *p = CONTAINER_FROM_VTBL(pp, CByteInToLook, vt);
+    CByteInToLook *p = Z7_CONTAINER_FROM_VTBL(pp, CByteInToLook, vt);
     if (p->cur != p->end) {
         return *p->cur++;
     }
@@ -404,7 +411,6 @@ pylzma_ppmd_decompress(PyObject *self, PyObject *args)
     unsigned order;
     UInt32 memSize;
     CPpmd7 ppmd;
-    CPpmd7z_RangeDec rc;
     CByteInToLook s;
     SRes res = SZ_OK;
     CMemoryLookInStream stream;
@@ -446,30 +452,29 @@ pylzma_ppmd_decompress(PyObject *self, PyObject *args)
     CreateMemoryLookInStream(&stream, (Byte*) data, length);
     tmp = (Byte *) PyBytes_AS_STRING(result);
     Py_BEGIN_ALLOW_THREADS
-    Ppmd7z_RangeDec_CreateVTable(&rc);
     s.vt.Read = ReadByte;
     s.inStream = &stream.s;
     s.begin = s.end = s.cur = NULL;
     s.extra = False;
     s.res = SZ_OK;
     s.processed = 0;
-    rc.Stream = &s.vt;
-    if (!Ppmd7z_RangeDec_Init(&rc)) {
+    ppmd.rc.dec.Stream = &s.vt;
+    if (!Ppmd7z_RangeDec_Init(&ppmd.rc.dec)) {
         res = SZ_ERROR_DATA;
     } else if (s.extra) {
         res = (s.res != SZ_OK ? s.res : SZ_ERROR_DATA);
     } else {
         SizeT i;
         for (i = 0; i < outsize; i++) {
-            int sym = Ppmd7_DecodeSymbol(&ppmd, &rc.vt);
+            int sym = Ppmd7z_DecodeSymbol(&ppmd);
             if (s.extra || sym < 0) {
                 break;
             }
             tmp[i] = (Byte)sym;
         }
         if (i != outsize) {
             res = (s.res != SZ_OK ? s.res : SZ_ERROR_DATA);
-        } else if (s.processed + (s.cur - s.begin) != (UInt64)length || !Ppmd7z_RangeDec_IsFinishedOK(&rc)) {
+        } else if (s.processed + (s.cur - s.begin) != (UInt64)length || !Ppmd7z_RangeDec_IsFinishedOK(&ppmd.rc.dec)) {
             res = SZ_ERROR_DATA;
         }
     }

src/pylzma/pylzma.c

@@ -1,8 +1,8 @@
 /* 7z.h -- 7z interface
-2017-04-03 : Igor Pavlov : Public domain */
+2023-04-02 : Igor Pavlov : Public domain */
 
-#ifndef __7Z_H
-#define __7Z_H
+#ifndef ZIP7_INC_7Z_H
+#define ZIP7_INC_7Z_H
 
 #include "7zTypes.h"
 
@@ -91,12 +91,14 @@ typedef struct
   UInt64 *CoderUnpackSizes;       // for all coders in all folders
 
   Byte *CodersData;
+
+  UInt64 RangeLimit;
 } CSzAr;
 
 UInt64 SzAr_GetFolderUnpackSize(const CSzAr *p, UInt32 folderIndex);
 
 SRes SzAr_DecodeFolder(const CSzAr *p, UInt32 folderIndex,
-    ILookInStream *stream, UInt64 startPos,
+    ILookInStreamPtr stream, UInt64 startPos,
     Byte *outBuffer, size_t outSize,
     ISzAllocPtr allocMain);
 
@@ -172,7 +174,7 @@ UInt16 *SzArEx_GetFullNameUtf16_Back(const CSzArEx *p, size_t fileIndex, UInt16
 
 SRes SzArEx_Extract(
     const CSzArEx *db,
-    ILookInStream *inStream,
+    ILookInStreamPtr inStream,
     UInt32 fileIndex,         /* index of file */
     UInt32 *blockIndex,       /* index of solid block */
     Byte **outBuffer,         /* pointer to pointer to output buffer (allocated with allocMain) */
@@ -194,7 +196,7 @@ SZ_ERROR_INPUT_EOF
 SZ_ERROR_FAIL
 */
 
-SRes SzArEx_Open(CSzArEx *p, ILookInStream *inStream,
+SRes SzArEx_Open(CSzArEx *p, ILookInStreamPtr inStream,
     ISzAllocPtr allocMain, ISzAllocPtr allocTemp);
 
 EXTERN_C_END