Issue 304 libzstd versions compiled and loaded

.github/workflows/Build_wheels_for_cpython27_x86_ext.yml

@@ -0,0 +1,128 @@
+name: Build wheels for CPython2.7 x86 on Ubuntu16 with GCC-5 with All warnings, test external build
+
+on: [push, pull_request]
+
+concurrency:
+  group: ${{ github.ref }}-${{ github.workflow }}-${{ github.event_name }}-${{ github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && github.sha || '' }}
+  cancel-in-progress: true
+
+jobs:
+  build_wheels:
+    name: Build wheel - ${{ matrix.os.name }}
+    runs-on: ${{ matrix.os.runs-on }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os:
+          - name: Ubuntu 24+16 i386 CPython 2.7
+            runs-on: ubuntu-latest
+            matrix: linux
+            arch: i386
+            tag_arch: i686
+            release: xenial
+            mirror: http://azure.archive.ubuntu.com/ubuntu
+            #version: 1.5.6.7
+            #pyver: "2.7"
+            getpipurl: https://bootstrap.pypa.io/pip/2.7/get-pip.py
+            pypkg: python2.7
+            pyengine_tag: cp27-cp27mu
+            libc_tag: manylinux_2_4
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          submodules: 'recursive'
+
+      - name: Update and upgrade Ubuntu 24
+        if: matrix.os.matrix == 'linux'
+        run: |
+          sudo which apt
+          sudo apt update; 
+          sudo apt purge -y firefox lxd snapd;
+          sudo apt install -y zram-config;
+          sudo apt list --upgradable;
+          sudo apt upgrade -y;
+          sudo apt install -f
+
+      - name: Build source distribution with Ubuntu
+        if: matrix.os.matrix == 'linux'
+        run: |
+          sudo apt install -y debootstrap qemu-user;
+          pip install build;
+          python -m build --sdist --outdir dist .
+
+      - name: Debootstrap Ubuntu ${{matrix.os.release}} ${{matrix.os.arch}}
+        if: matrix.os.matrix == 'linux'
+        run: |
+          sudo debootstrap --no-merged-usr --verbose --include=sudo,wget,curl,gnupg,ca-certificates --arch=${{matrix.os.arch}} ${{matrix.os.release}} ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}} ${{matrix.os.mirror}} || tail ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}}/debootstrap/debootstrap.log
+
+      - name: Update and Upgrade Ubuntu ${{matrix.os.release}}
+        if: matrix.os.matrix == 'linux'
+        run: |
+          sudo mkdir -p ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}}/build_wheel
+          sudo mount none ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}}/proc -t proc
+          sudo mount none ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}}/dev/pts -t devpts
+          sudo mount none ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}}/sys -t sysfs
+          echo " deb [trusted=yes] http://packages.rusoft.ru/ppa/rusoft/python ubuntu-${{matrix.os.release}} main" > rusoft-python.list
+          echo " deb [trusted=yes] http://packages.rusoft.ru/ppa/rusoft/backports ubuntu-${{matrix.os.release}} main" > rusoft-backports.list
+          echo " deb [trusted=yes] http://packages.rusoft.ru/ppa/rusoft/packages ubuntu-${{matrix.os.release}} main" > rusoft-packages.list
+          echo "# deb [trusted=yes] http://ppa.launchpadcontent.net/deadsnakes/ppa/ubuntu ${{matrix.os.release}} main" > deadsnakes.list
+          echo "# deb [trusted=yes] http://apt.llvm.org/${{matrix.os.release}}/ llvm-toolchain-${{matrix.os.release}}-20 main " > clang.list
+          sudo chroot ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}} /usr/bin/apt update
+          sudo cp -v ./*.list ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}}/etc/apt/sources.list.d
+          sudo chroot ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}} /bin/bash -c 'cd /etc/apt/trusted.gpg.d && wget http://packages.rusoft.ru/apt/public.gpg -Orusoft.gpg'
+          sudo chroot ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}} /bin/bash -c 'cd /etc/apt/trusted.gpg.d && wget http://packages.rusoft.ru/apt/public-old.gpg -Orusoft-old.gpg'
+          sudo chroot ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}} /bin/bash -c 'apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys A2CE4BCCC50209DD || true'
+          sudo chroot ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}} /usr/bin/apt update
+          sudo chroot ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}} /usr/bin/apt dist-upgrade -y
+          sudo chroot ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}} /usr/bin/apt -fy install
+          sudo chroot ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}} /usr/bin/apt upgrade -y
+
+      - name: Pepare dev files
+        if: matrix.os.matrix == 'linux'
+        run: |
+           sudo chroot ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}} /usr/bin/apt-cache search libzstd-dev
+           sudo chroot ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}} /usr/bin/apt-cache madison libzstd
+           sudo chroot ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}} /usr/bin/apt install -y libzstd-dev
+
+      - name: Pepare for ${{matrix.os.pypkg}} ${{matrix.os.arch}}
+        if: matrix.os.matrix == 'linux'
+        run: |
+           sudo chroot ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}} /usr/bin/apt-cache madison clang-18
+           sudo chroot ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}} /usr/bin/apt install -y ${{matrix.os.pypkg}}-dev gcc pkg-config ${{matrix.os.pypkgadd}}
+           sudo chroot ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}} /bin/bash -c "wget ${{matrix.os.getpipurl}} -O get-pip.py && ${{matrix.os.pypkg}} get-pip.py"
+           sudo chroot ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}} /bin/bash -c "${{matrix.os.pypkg}} -m pip install -U 'setuptools'" 
+
+      - name: Pepare source and build wheel for ${{matrix.os.pypkg}} ${{matrix.os.arch}}
+        if: matrix.os.matrix == 'linux'
+        run: |
+          sudo mkdir -p ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}}/build_wheel/
+          version=`cat version`
+          echo "$version"
+          sudo tar -xvf dist/zstd-$version.tar.gz -C ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}}/build_wheel/
+          false && sudo chroot ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}} env
+
+          sudo chroot ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}} pkg-config libzstd --modversion 
+          sudo chroot ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}} echo GCC-5
+          sudo chroot ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}} gcc-5 -v
+          sudo chroot ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}} /bin/bash -c "cd /build_wheel/zstd-$version && _CC=clang-20 ZSTD_ASM=1 ZSTD_WARNINGS=1 ZSTD_WERRORS=1 _ZSTD_SMALL=1 _ZSTD_ASM_BMI2=1 ZSTD_THREADS=1 ZSTD_EXTERNAL=1 CC=gcc-5 ${{matrix.os.pypkg}} setup.py bdist_wheel "
+
+      - name: Test wheel for ${{matrix.os.pypkg}} ${{matrix.os.arch}}
+        if: matrix.os.matrix == 'linux'
+        run: |
+          false && sudo chroot ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}} find /build_wheel/
+          version=`cat version`
+          echo version=$version
+          sudo cp -v glibc-check.sh ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}}/build_wheel/zstd-$version/
+          sudo chroot ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}} /bin/bash -c "cd /build_wheel/zstd-$version/ ; echo LLH; ls -lh build/*/zstd.so; echo LDD; ldd build/*/zstd.so;echo FILE; file build/*/zstd.so; echo LIBC; bash ./glibc-check.sh build/*/zstd.so"
+          sudo chroot ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}} /bin/bash -c "cd /build_wheel/zstd-$version && ${{matrix.os.pypkg}} setup.py test"
+
+      - name: Pepare wheel for upload
+        if: matrix.os.matrix == 'linux'
+        run: |
+           version=`cat version`
+           echo version=$version
+           sudo cp -v ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}}/build_wheel/zstd-$version/dist/* ./dist
+           sudo ls -lh ./dist/* 
+           sudo mv -v ./dist/"zstd-$version-${{matrix.os.pyengine_tag}}-linux_${{matrix.os.tag_arch}}.whl" ./dist/"zstd-$version-${{matrix.os.pyengine_tag}}-${{matrix.os.libc_tag}}_${{matrix.os.tag_arch}}.whl"
+

README.rst

@@ -23,6 +23,9 @@ python-zstd
 .. |cpython27x86| image:: https://github.com/sergey-dryabzhinsky/python-zstd/actions/workflows/Build_wheels_for_cpython27_x86.yml/badge.svg
     :target: https://github.com/sergey-dryabzhinsky/python-zstd/actions/workflows/Build_wheels_for_cpython27_x86.yml
 
+.. |cpython27x86ext| image:: https://github.com/sergey-dryabzhinsky/python-zstd/actions/workflows/Build_wheels_for_cpython27_x86_ext.yml/badge.svg
+    :target: https://github.com/sergey-dryabzhinsky/python-zstd/actions/workflows/Build_wheels_for_cpython27_x86_ext.yml
+
 .. |cpython27armhf| image:: https://github.com/sergey-dryabzhinsky/python-zstd/actions/workflows/Build_wheels_for_cpython27_armhf.yml/badge.svg
     :target: https://github.com/sergey-dryabzhinsky/python-zstd/actions/workflows/Build_wheels_for_cpython27_armhf.yml
 
@@ -123,6 +126,8 @@ python-zstd
 +============================+======================+
 | cpython 2.7 x86 u18        | |cpython27x86|       |
 +----------------------------+----------------------+
+| cpython 2.7 x86 u18 ext    | |cpython27x86ext|    |
++----------------------------+----------------------+
 | cpython 2.7 x64 u18        | |cpython27x64|       |
 +----------------------------+----------------------+
 | cpython 2.7 armhf u18      | |cpython27armhf|     |
@@ -496,6 +501,16 @@ ZSTD_version (): string|bytes
 
   Since: 1.3.4.3
 
+ZSTD_version_compiled (): string|bytes
+  Returns ZSTD library doted version string, wi which are compiled.
+
+  Since: 1.5.7.3
+
+ZSTD_version_loaded (): string|bytes
+  Returns ZSTD library doted version string, with which currently loaded.
+
+  Since: 1.5.7.3
+
 ZSTD_version_number (): int
   Returns ZSTD library version in format: MAJOR*100*100 + MINOR*100 + RELEASE.
 

setup.py

@@ -264,7 +264,7 @@ def which(bin_exe):
     pkgconf = False
 
 #if SUP_EXTERNAL:
-if platform.system() == "Linux" and "build_ext" in sys.argv or "build" in sys.argv or "bdist_wheel" in sys.argv:
+if platform.system() == "Linux" and "build_ext" in sys.argv or "build" in sys.argv or "bdist_wheel" in sys.argv or "test" in sys.argv:
     # You should add external library by option: --libraries zstd
     # And probably include paths by option: --include-dirs /usr/include/zstd
     # And probably library paths by option: --library-dirs /usr/lib/i386-linux-gnu
@@ -521,7 +521,7 @@ def build_extensions(self):
             'compress/zstd_preSplit.c',
             'compress/zstdmt_compress.c',
             'compress/zstd_fast.c',
-            'compress/zstd_double_fast.c',
+#            'compress/zstd_double_fast.c',
             'compress/zstd_lazy.c',
             'compress/zstd_opt.c',
             'compress/zstd_ldm.c',
@@ -550,10 +550,16 @@ def build_extensions(self):
             ]:
             zstdFiles.append('zstd/lib/'+f)
 
+# files needed always, even for external
 zstdFiles.append('src/debug.c')
 zstdFiles.append('src/util.c')
 zstdFiles.append('src/python-zstd.c')
 
+#for f in [
+#        'decompress/zstd_decompress.c',
+#        'decompress/zstd_ddict.c',
+#    ]:
+#    zstdFiles.append('zstd/lib/'+f)
 
 # Another dirty hack
 def my_test_suite():
@@ -580,6 +586,8 @@ def my_test_suite():
 ld=f.read()
 f.close()
 
+if SUP_DEBUG:
+	print("debug: ext_libraries:%r" % (ext_libraries))
 setup(
     name='zstd',
     version=PKG_VERSION_STR,

src/python-zstd.c

@@ -329,9 +329,24 @@ static PyObject *py_zstd_module_version(PyObject* self, PyObject *args)
 }
 
 /**
- * Returns ZSTD library version as string
+ * Returns ZSTD library version as string - compiled with
  */
-static PyObject *py_zstd_library_version(PyObject* self, PyObject *args)
+static PyObject *py_zstd_library_version_compiled(PyObject* self, PyObject *args)
+{
+     UNUSED(self);
+     UNUSED(args);
+
+#if PY_MAJOR_VERSION >= 3
+    return PyUnicode_FromFormat("%s", ZSTD_VERSION_STRING);
+#else
+    return PyString_FromFormat("%s", ZSTD_VERSION_STRING);
+#endif
+}
+
+/**
+ * Returns ZSTD library version as string - loaded with
+ */
+static PyObject *py_zstd_library_version_loaded(PyObject* self, PyObject *args)
 {
      UNUSED(self);
      UNUSED(args);
@@ -545,7 +560,9 @@ static PyMethodDef ZstdMethods[] = {
     {"dumps",  py_zstd_compress_mt, METH_VARARGS, COMPRESS_DOCSTRING},
     {"loads",  py_zstd_uncompress, METH_VARARGS, UNCOMPRESS_DOCSTRING},
     {"version",  py_zstd_module_version, METH_NOARGS, VERSION_DOCSTRING},
-    {"ZSTD_version",  py_zstd_library_version, METH_NOARGS, ZSTD_VERSION_DOCSTRING},
+    {"ZSTD_version",  py_zstd_library_version_compiled, METH_NOARGS, ZSTD_VERSION_DOCSTRING},
+    {"ZSTD_version_compiled",  py_zstd_library_version_compiled, METH_NOARGS, NULL},
+    {"ZSTD_version_loaded",  py_zstd_library_version_loaded, METH_NOARGS, NULL},
     {"ZSTD_version_number",  py_zstd_library_version_int, METH_NOARGS, ZSTD_INT_VERSION_DOCSTRING},
     {"ZSTD_threads_count",  py_zstd_threads_count, METH_NOARGS, ZSTD_THREADS_COUNT_DOCSTRING},
     {"ZSTD_max_threads_count",  py_zstd_max_threads_count, METH_NOARGS, ZSTD_MAX_THREADS_COUNT_DOCSTRING},

src/python-zstd.h

@@ -96,7 +96,8 @@ static PyObject *py_zstd_compress_mt(PyObject* self, PyObject *args);
 static PyObject *py_zstd_uncompress(PyObject* self, PyObject *args);
 static PyObject *py_zstd_check(PyObject* self, PyObject *args);
 static PyObject *py_zstd_module_version(PyObject* self, PyObject *args);
-static PyObject *py_zstd_library_version(PyObject* self, PyObject *args);
+static PyObject *py_zstd_library_version_compiled(PyObject* self, PyObject *args);
+static PyObject *py_zstd_library_version_loaded(PyObject* self, PyObject *args);
 static PyObject *py_zstd_library_version_int(PyObject* self, PyObject *args);
 static PyObject *py_zstd_library_external(PyObject* self, PyObject *args);
 static PyObject *py_zstd_with_threads(PyObject* self, PyObject *args);

tests/base.py

@@ -22,16 +22,18 @@ def raise_skip(msg):
 log = logging.getLogger('ZSTD')
 log.info("Python version: %s" % sys.version)
 log.info("Machine:%s; processor:%s; system:%r; release:%r" % ( platform.machine(), platform.processor(), platform.system(), platform.release()))
-log.info("Module exported interface:%r"% dir(zstd))
-log.info("libzstd linked external:%r"% zstd.ZSTD_external())
-log.info("libzstd built with legacy formats support:%r"% zstd.ZSTD_legacy_support())
-log.info("zstd max number of threads:%r"% zstd.ZSTD_max_threads_count())
-log.info("zstd found CPU cores :%r"% zstd.ZSTD_threads_count())
-log.info("zstd default compression level:%r"% zstd.ZSTD_default_compression_level())
-log.info("zstd max compression level:%r"% zstd.ZSTD_max_compression_level())
-log.info("zstd min compression level:%r"% zstd.ZSTD_min_compression_level())
+log.info("Module exported interfac e:%r"% dir(zstd))
+log.info("libzstd linked external: %r"% zstd.ZSTD_external())
+log.info("libzstd built with legacy formats support: %r"% zstd.ZSTD_legacy_support())
+log.info("zstd max number of threads: %r"% zstd.ZSTD_max_threads_count())
+log.info("zstd found CPU cores : %r"% zstd.ZSTD_threads_count())
+log.info("zstd default compression level: %r"% zstd.ZSTD_default_compression_level())
+log.info("zstd max compression level: %r"% zstd.ZSTD_max_compression_level())
+log.info("zstd min compression level: %r"% zstd.ZSTD_min_compression_level())
 log.info("pyzstd module version: %r"% zstd.version())
 log.info("libzstd version: %r"% zstd.ZSTD_version())
+log.info("Compiled with libzstd version: %r"% zstd.ZSTD_version_compiled())
+log.info("Loaded with libzstd version: %r"% zstd.ZSTD_version_loaded())
 
 
 # Classic lorem ipsum