Issue 301 initialize compress2 context only once

.github/workflows/Build_wheels_for_cpython313_x86_64_u24.yml

@@ -1,4 +1,4 @@
-name: Build wheels for CPython3.13 x86_64 on Ubuntu16 with GCC-5 All warnings
+name: Build wheels for CPython3.13 x86_64 on Ubuntu22 with GCC-10 All warnings
 
 on: [push, pull_request]
 
@@ -14,18 +14,15 @@ jobs:
       fail-fast: false
       matrix:
         os:
-          - name: Ubuntu 20 amd64 CPython 3.13
+          - name: Ubuntu 24+20 amd64 CPython 3.13
             runs-on: ubuntu-latest
             matrix: linux
             arch: amd64
             tag_arch: x86_64
-            release: xenial
+            release: jammy
             mirror: http://azure.archive.ubuntu.com/ubuntu
-            #version: 1.5.6.7
-            # pyver: "3.7"
             getpipurl: https://bootstrap.pypa.io/pip/get-pip.py
             pypkg: python3.13
-            #pypkgadd: python3.13-distutils
             pyengine_tag: cp313-cp313
             libc_tag: manylinux_2_14
     steps:
@@ -64,9 +61,9 @@ jobs:
           sudo mount none ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}}/proc -t proc
           sudo mount none ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}}/dev/pts -t devpts
           sudo mount none ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}}/sys -t sysfs
-          echo " deb [trusted=yes] http://packages.rusoft.ru/ppa/rusoft/python ubuntu-${{matrix.os.release}} main" > rusoft-python.list
-          echo " deb [trusted=yes] http://packages.rusoft.ru/ppa/rusoft/backports ubuntu-${{matrix.os.release}} main" > rusoft-backports.list
-          echo " deb [trusted=yes] http://packages.rusoft.ru/ppa/rusoft/packages ubuntu-${{matrix.os.release}} main" > rusoft-packages.list
+          echo "# deb [trusted=yes] http://packages.rusoft.ru/ppa/rusoft/python ubuntu-${{matrix.os.release}} main" > rusoft-python.list
+          echo "# deb [trusted=yes] http://packages.rusoft.ru/ppa/rusoft/backports ubuntu-${{matrix.os.release}} main" > rusoft-backports.list
+          echo "# deb [trusted=yes] http://packages.rusoft.ru/ppa/rusoft/packages ubuntu-${{matrix.os.release}} main" > rusoft-packages.list
           echo " deb [trusted=yes] http://ppa.launchpadcontent.net/deadsnakes/ppa/ubuntu ${{matrix.os.release}} main" > deadsnakes.list
           false && sudo find ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}} -iname apt
           sudo chroot ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}} /usr/bin/apt update

.github/workflows/Build_wheels_for_cpython313_x86_u16.yml

@@ -1,4 +1,4 @@
-name: Build wheels for CPython3.13 x86 on Ubuntu16 with GCC-5 All warnings
+name: Build wheels for CPython3.13 x86 on Ubuntu22 with GCC-10 All warnings
 
 on: [push, pull_request]
 
@@ -14,18 +14,15 @@ jobs:
       fail-fast: false
       matrix:
         os:
-          - name: Ubuntu 24+16 x86 CPython 3.13
+          - name: Ubuntu 24+22 x86 CPython 3.13
             runs-on: ubuntu-latest
             matrix: linux
             arch: i386
             tag_arch: i686
-            release: xenial
+            release: jammy
             mirror: http://azure.archive.ubuntu.com/ubuntu
-            #version: 1.5.6.7
-            # pyver: "3.7"
             getpipurl: https://bootstrap.pypa.io/pip/get-pip.py
             pypkg: python3.13
-            #pypkgadd: python3.13-distutils
             pyengine_tag: cp313-cp313
             libc_tag: manylinux_2_14
     steps:
@@ -63,9 +60,9 @@ jobs:
           sudo mount none ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}}/proc -t proc
           sudo mount none ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}}/dev/pts -t devpts
           sudo mount none ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}}/sys -t sysfs
-          echo " deb [trusted=yes] http://packages.rusoft.ru/ppa/rusoft/python ubuntu-${{matrix.os.release}} main" > rusoft-python.list
-          echo " deb [trusted=yes] http://packages.rusoft.ru/ppa/rusoft/backports ubuntu-${{matrix.os.release}} main" > rusoft-backports.list
-          echo " deb [trusted=yes] http://packages.rusoft.ru/ppa/rusoft/packages ubuntu-${{matrix.os.release}} main" > rusoft-packages.list
+          echo "# deb [trusted=yes] http://packages.rusoft.ru/ppa/rusoft/python ubuntu-${{matrix.os.release}} main" > rusoft-python.list
+          echo "# deb [trusted=yes] http://packages.rusoft.ru/ppa/rusoft/backports ubuntu-${{matrix.os.release}} main" > rusoft-backports.list
+          echo "# deb [trusted=yes] http://packages.rusoft.ru/ppa/rusoft/packages ubuntu-${{matrix.os.release}} main" > rusoft-packages.list
           echo " deb [trusted=yes] http://ppa.launchpadcontent.net/deadsnakes/ppa/ubuntu ${{matrix.os.release}} main" > deadsnakes.list
           sudo chroot ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}} /usr/bin/apt update
           sudo cp -v ./*.list ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}}/etc/apt/sources.list.d

.github/workflows/Build_wheels_for_cpython313t_x64_u16.yml

@@ -1,4 +1,4 @@
-name: Build wheels for CPython3.13-nogil x64 on Ubuntu16 with GCC-5 All warnings
+name: Build wheels for CPython3.13-nogil x64 on Ubuntu22 with GCC-10 All warnings
 
 on: [push, pull_request]
 
@@ -14,20 +14,17 @@ jobs:
       fail-fast: false
       matrix:
         os:
-          - name: Ubuntu 24+16 amd64 CPython 3.13-nogil
+          - name: Ubuntu 24+22 amd64 CPython 3.13-nogil
             runs-on: ubuntu-latest
             matrix: linux
             arch: amd64
             tag_arch: x86_64
-            release: xenial
+            release: jammy
             mirror: http://azure.archive.ubuntu.com/ubuntu
-            #version: 1.5.6.7
-            # pyver: "3.7"
             getpipurl: https://bootstrap.pypa.io/pip/get-pip.py
             pypkg: python3.13-nogil
             pypkgdev: python3.13-dev
             pypkgexe: python3.13t
-            #pypkgadd: python3.13-distutils
             pyengine_tag: cp313-cp313t
             libc_tag: manylinux_2_14
     steps:
@@ -66,14 +63,12 @@ jobs:
           sudo mount none ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}}/proc -t proc
           sudo mount none ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}}/dev/pts -t devpts
           sudo mount none ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}}/sys -t sysfs
-          echo " deb [trusted=yes] http://packages.rusoft.ru/ppa/rusoft/python ubuntu-${{matrix.os.release}} main" > rusoft-python.list
-          echo " deb [trusted=yes] http://packages.rusoft.ru/ppa/rusoft/backports ubuntu-${{matrix.os.release}} main" > rusoft-backports.list
-          echo " deb [trusted=yes] http://packages.rusoft.ru/ppa/rusoft/packages ubuntu-${{matrix.os.release}} main" > rusoft-packages.list
+          echo "# deb [trusted=yes] http://packages.rusoft.ru/ppa/rusoft/python ubuntu-${{matrix.os.release}} main" > rusoft-python.list
+          echo "# deb [trusted=yes] http://packages.rusoft.ru/ppa/rusoft/backports ubuntu-${{matrix.os.release}} main" > rusoft-backports.list
+          echo "# deb [trusted=yes] http://packages.rusoft.ru/ppa/rusoft/packages ubuntu-${{matrix.os.release}} main" > rusoft-packages.list
           echo " deb [trusted=yes] http://ppa.launchpadcontent.net/deadsnakes/ppa/ubuntu ${{matrix.os.release}} main" > deadsnakes.list
-          false && sudo find ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}} -iname apt
           sudo chroot ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}} /usr/bin/apt update
           sudo cp -v ./*.list ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}}/etc/apt/sources.list.d
-          false && sudo rm -fv ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}}/etc/apt/trusted.gpg.d/* && sudo rm -fv ./ubuntu-xenial-x86/etc/apt/trusted.gpg
           sudo chroot ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}} /bin/bash -c 'cd /etc/apt/trusted.gpg.d && wget http://packages.rusoft.ru/apt/public.gpg -Orusoft.gpg'
           sudo chroot ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}} /bin/bash -c 'cd /etc/apt/trusted.gpg.d && wget http://packages.rusoft.ru/apt/public-old.gpg -Orusoft-old.gpg'
           sudo chroot ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}} /bin/bash -c 'apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys A2CE4BCCC50209DD || true'
@@ -101,11 +96,10 @@ jobs:
           version=`cat version`
           sudo mkdir -p ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}}/build_wheel/
           sudo tar -xvf dist/zstd-$version.tar.gz -C ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}}/build_wheel/
-          false && sudo chroot ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}} env 
           sudo chroot ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}} pkg-config libzstd --modversion 
           sudo chroot ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}} echo GCC
           sudo chroot ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}} gcc -v
-          sudo chroot ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}} /bin/bash -c "cd /build_wheel/zstd-$version/ && ZSTD_ASM=1 ZSTD_WARNINGS=1 _ZSTD_SMALL=1 LDFLAGS='-Wl,--as-needed -Wl,-s' ZSTD_BUILD_STRIP=1 _ZSTD_SPEED=0 _ZSTD_SPEEDMAX=0 _ZSTD_ASM_BMI2=1 ZSTD_THREADS=1 _ZSTD_EXTERNAL=0 ${{matrix.os.pypkgexe}} setup.py bdist_wheel"
+          sudo chroot ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}} /bin/bash -c "cd /build_wheel/zstd-$version/ && ZSTD_ASM=1 ZSTD_WARNINGS=1 _ZSTD_SMALL=1 LDFLAGS='-Wl,--as-needed -Wl,-s' ZSTD_BUILD_STRIP=1 _ZSTD_SPEED=0 _ZSTD_SPEEDMAX=0 _ZSTD_ASM_BMI2=1 ZSTD_THREADS=1 _ZSTD_EXTERNAL=0 ZS5D_WERRORS=1 ${{matrix.os.pypkgexe}} setup.py bdist_wheel"
 
       - name: Test wheel for ${{matrix.os.pypkg}} ${{matrix.os.arch}}
         if: matrix.os.matrix == 'linux'

MANIFEST.in

@@ -6,5 +6,6 @@ graft zstd/lib/legacy
 graft tests
 include LICENSE
 include README.rst
+include version
 include zstd/lib/zstd.h
 include zstd/lib/zstd_errors.h

setup.py

@@ -21,9 +21,17 @@
 # Package version, even external 
 PKG_VERSION = VERSION
 # Minor revision 
-PKG_VERSION += ("2",)
+PKG_VERSION += (2,)
 PKG_VERSION_STR = ".".join([str(x) for x in PKG_VERSION])
-
+vpath="./version"
+if not os.path.exists(vpath):
+	vdn=os.path.dirname( os.path.abspath(sys.argv[0]))
+	vpath=os.path.join(vdn,vpath)
+if os.path.exists(vpath):
+	print("use 3xisting %s file" % vpath)
+	f = open(vpath,"r" )
+PKG_VERSION_STR = f.read().strip()
+f.close()
 
 def which(bin_exe):
     """
@@ -60,7 +68,7 @@ def which(bin_exe):
 SUP_WARNINGS="ZSTD_WARNINGS" in os.environ
 if SUP_WARNINGS:
     if os.environ["ZSTD_WARNINGS"]=='0':
-        SUP_WARNINS=False
+        SUP_WARNINGS=False
     else:
         SUP_WARNINGS=True
 if "--all-warnings" in sys.argv:
@@ -119,6 +127,7 @@ def which(bin_exe):
     # Support tracing for debug
     SUP_TRACE=True
     sys.argv.remove("--debug-trace")
+
 BUILD_SPEED0="ZSTD_SPEED0" in os.environ
 if "--speed0" in sys.argv:
     # speed or size choose only one
@@ -455,9 +464,7 @@ def which(bin_exe):
         if comp == 'msvc':
             COPT[comp].extend(['/Wall',])
         else:
-            COPT[comp].extend(['-Wall', '-Wextra', 
-#'-Wpedantic'
-])
+            COPT[comp].extend(['-Wall', '-Wextra', '-Wpedantic'])
 
 if SUP_WERROR:
     for comp in COPT:

src/python-zstd.c

@@ -48,7 +48,8 @@
 #include "python-zstd.h"
 
 /**
- * New function for multi-threaded compression.
+ * @deprecated, use *mt2 instead.
+ * Old function for multi-threaded compression.
  * Uses origin zstd header, nothing more.
  * Simple version: not for streaming, no dict support, full block compression.
  * Uses new API with context object.
@@ -159,6 +160,135 @@ static PyObject *py_zstd_compress_mt(PyObject* self, PyObject *args)
     return result;
 }
 
+void init_cContext( int32_t threads, int32_t level)
+{
+    m_cctx = ZSTD_createCCtx();
+    ZSTD_CCtx_setParameter(m_cctx, ZSTD_c_compressionLevel, level);
+    ZSTD_CCtx_setParameter(m_cctx, ZSTD_c_nbWorkers, threads);
+}
+
+void free_cContext(void)
+{
+    ZSTD_freeCCtx(m_cctx);
+}
+
+void reset_cContext(int32_t threads, int32_t level)
+{
+	free_cContext();
+	init_cContext(threads, level);
+}
+
+/**
+ * New function for multi-threaded compression.
+ * Uses origin zstd header, nothing more.
+ * Simple version: not for streaming, no dict support, full block compression.
+ * Uses new API with context object, initialize only once.
+ */
+static PyObject *py_zstd_compress_mt2(PyObject* self, PyObject *args)
+{
+    UNUSED(self);
+
+    PyObject *result;
+//    PyObject *resultT;
+    const char *source;
+    Py_ssize_t source_size;
+//    Py_ssize_t chunk_size;
+    char *dest;
+//    char *destT;
+    Py_ssize_t dest_size;
+    size_t cSize;
+//    size_t sum=0;
+    int32_t level = ZSTD_CLEVEL_DEFAULT;
+	static int32_t lastLevel =0;
+    int32_t threads = 0;
+    int32_t strict = 0;
+
+#if PY_MAJOR_VERSION >= 3
+    if (!PyArg_ParseTuple(args, "y#|iii", &source, &source_size, &level, &threads, &strict))
+        return NULL;
+#else
+    if (!PyArg_ParseTuple(args, "s#|iii", &source, &source_size, &level, &threads, &strict))
+        return NULL;
+#endif
+
+    printdn("got Compression level:%d\n",level);
+    if (0 == level) level=ZSTD_defaultCLevel();
+    /* Fast levels (zstd >= 1.3.4) - [-1..-100] */
+    /* Usual levels                - [ 1..22] */
+    /* If level less than -100 or 1 - raise Error, level 0 handled before. */
+    printdn("Compression min level:%d\n",ZSTD_MIN_CLEVEL);
+    if (level < ZSTD_MIN_CLEVEL) {
+        printd2e("Bad compression level - less than %d: %d\n", ZSTD_MIN_CLEVEL, level);
+	if (strict) {
+        PyErr_Format(ZstdError, "Bad compression level - less than %d: %d", ZSTD_MIN_CLEVEL, level);
+        return NULL;
+	} else {
+	    level = ZSTD_MIN_CLEVEL;
+	}
+    }
+    /* If level more than 22 - raise Error. */
+    printdn("Compression max level:%d\n",ZSTD_maxCLevel());
+    if (level > ZSTD_maxCLevel()) {
+        printd2e("Bad compression level - more than %d: %d\n", ZSTD_maxCLevel(), level);
+	if (strict) {
+        PyErr_Format(ZstdError, "Bad compression level - more than %d: %d", ZSTD_MAX_CLEVEL, level);
+        return NULL;
+	} else {
+	    level = ZSTD_maxCLevel();
+	}
+    }
+    printdn("Compression level will be:%d\n",level);
+
+    printdn("got Compression threads:%d\n",threads);
+    if (threads < 0) {
+        printd2e("Bad threads count - less than %d: %d\n", 0, threads);
+	if (strict) {
+        PyErr_Format(ZstdError, "Bad threads count - less than %d: %d", 0, threads);
+        return NULL;
+	} else threads = 1;
+    }
+    if (0 == threads) threads = UTIL_countAvailableCores();
+    printdn("got CPU cores:%d\n",threads);
+    /* If threads more than 200 - raise Error. */
+    if (threads > ZSTDMT_NBWORKERS_MAX) {
+        printd2e("Bad threads count - more than %d: %d\n", ZSTDMT_NBWORKERS_MAX, threads);
+        threads = ZSTDMT_NBWORKERS_MAX;
+        // do not fail here, due auto thread counter
+        //PyErr_Format(ZstdError, "Bad threads count - more than %d: %d", ZSTDMT_NBWORKERS_MAX, threads);
+        //return NULL;
+    }
+    printdn("Compression will use:%d threads\n",threads);
+
+    dest_size = (Py_ssize_t)ZSTD_compressBound(source_size);
+    result = PyBytes_FromStringAndSize(NULL, dest_size);
+    if (result == NULL) {
+        return NULL;
+    }
+
+    if (source_size >= 0) {
+        dest = PyBytes_AS_STRING(result);
+
+		if(level != lastLevel) {
+			reset_cContext(threads, level);
+		}
+
+        Py_BEGIN_ALLOW_THREADS
+        cSize = ZSTD_compress2(m_cctx, dest, (size_t)dest_size, source, (size_t)source_size);
+        Py_END_ALLOW_THREADS
+		lastLevel = level;
+
+        printdn("Compression result: %d\n", cSize);
+        if (ZSTD_isError(cSize)) {
+            printdes("debug INFO: Compression error: %s", ZSTD_getErrorName(cSize));
+            PyErr_Format(ZstdError, "Compression error: %s", ZSTD_getErrorName(cSize));
+            Py_CLEAR(result);
+            return NULL;
+        }
+        Py_SET_SIZE(result, cSize);
+    }
+    return result;
+}
+
 
 /**
  * New more interoperable function
@@ -531,12 +661,14 @@ static PyObject *py_zstd_max_compression_level(PyObject* self, PyObject *args)
 
 
 static PyMethodDef ZstdMethods[] = {
-    {"ZSTD_compress",  py_zstd_compress_mt, METH_VARARGS, COMPRESS_DOCSTRING},
+    {"ZSTD_compress",  py_zstd_compress_mt2, METH_VARARGS, COMPRESS_DOCSTRING},
     {"ZSTD_uncompress",  py_zstd_uncompress, METH_VARARGS, UNCOMPRESS_DOCSTRING},
     {"ZSTD_check",  py_zstd_check, METH_VARARGS, CHECK_DOCSTRING},
     {"check",  py_zstd_check, METH_VARARGS, CHECK_DOCSTRING},
     {"verify",  py_zstd_check, METH_VARARGS, CHECK_DOCSTRING},
+    {"compress_old",  py_zstd_compress_mt, METH_VARARGS, COMPRESS_DOCSTRING},
     {"compress",  py_zstd_compress_mt, METH_VARARGS, COMPRESS_DOCSTRING},
+    {"compress2",  py_zstd_compress_mt2, METH_VARARGS, COMPRESS_DOCSTRING},
     {"compress_real_mt",  py_zstd_compress_mt, METH_VARARGS, COMPRESS_DOCSTRING},
     {"uncompress",  py_zstd_uncompress, METH_VARARGS, UNCOMPRESS_DOCSTRING},
     {"encode",  py_zstd_compress_mt, METH_VARARGS, COMPRESS_DOCSTRING},
@@ -586,6 +718,7 @@ static int init_py_zstd(PyObject *module) {
     Py_INCREF(ZstdError);
     PyModule_AddObject(module, "Error", ZstdError);
 
+	init_cContext(1, 3);
     return 0;
 }
 
@@ -611,6 +744,7 @@ static int myextension_clear(PyObject *self) {
 
 static void myextension_free(void *self) {
     Py_CLEAR(GETSTATE((PyObject *)self)->error);
+	free_cContext();
     printdi("ZSTD module->free\n",0);
     return;
 }