Skip to content

Commit 0ecb3e5

Browse files
sergey-dryabzhinskysergey-dryabzhinsky
authored
Merge pull request #306 from sergey-dryabzhinsky/issue-304-libzstd-versions-compiled-and-loaded
Issue 304 libzstd versions compiled and loaded
2 parents d7cd3fe + 5711e8e commit 0ecb3e5

File tree

6 files changed

+185
-14
lines changed

6 files changed

+185
-14
lines changed

‎.github/workflows/Build_wheels_for_cpython27_x86_ext.yml‎

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
name: Build wheels for CPython2.7 x86 on Ubuntu16 with GCC-5 with All warnings, test external build
2+
3+
on: [push, pull_request]
4+
5+
concurrency:
6+
group: ${{ github.ref }}-${{ github.workflow }}-${{ github.event_name }}-${{ github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && github.sha || '' }}
7+
cancel-in-progress: true
8+
9+
jobs:
10+
build_wheels:
11+
name: Build wheel - ${{ matrix.os.name }}
12+
runs-on: ${{ matrix.os.runs-on }}
13+
strategy:
14+
fail-fast: false
15+
matrix:
16+
os:
17+
- name: Ubuntu 24+16 i386 CPython 2.7
18+
runs-on: ubuntu-latest
19+
matrix: linux
20+
arch: i386
21+
tag_arch: i686
22+
release: xenial
23+
mirror: http://azure.archive.ubuntu.com/ubuntu
24+
#version: 1.5.6.7
25+
#pyver: "2.7"
26+
getpipurl: https://bootstrap.pypa.io/pip/2.7/get-pip.py
27+
pypkg: python2.7
28+
pyengine_tag: cp27-cp27mu
29+
libc_tag: manylinux_2_4
30+
steps:
31+
- name: Checkout code
32+
uses: actions/checkout@v4
33+
with:
34+
submodules: 'recursive'
35+
36+
- name: Update and upgrade Ubuntu 24
37+
if: matrix.os.matrix == 'linux'
38+
run: |
39+
sudo which apt
40+
sudo apt update;
41+
sudo apt purge -y firefox lxd snapd;
42+
sudo apt install -y zram-config;
43+
sudo apt list --upgradable;
44+
sudo apt upgrade -y;
45+
sudo apt install -f
46+
47+
- name: Build source distribution with Ubuntu
48+
if: matrix.os.matrix == 'linux'
49+
run: |
50+
sudo apt install -y debootstrap qemu-user;
51+
pip install build;
52+
python -m build --sdist --outdir dist .
53+
54+
- name: Debootstrap Ubuntu ${{matrix.os.release}} ${{matrix.os.arch}}
55+
if: matrix.os.matrix == 'linux'
56+
run: |
57+
sudo debootstrap --no-merged-usr --verbose --include=sudo,wget,curl,gnupg,ca-certificates --arch=${{matrix.os.arch}} ${{matrix.os.release}} ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}} ${{matrix.os.mirror}} || tail ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}}/debootstrap/debootstrap.log
58+
59+
- name: Update and Upgrade Ubuntu ${{matrix.os.release}}
60+
if: matrix.os.matrix == 'linux'
61+
run: |
62+
sudo mkdir -p ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}}/build_wheel
63+
sudo mount none ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}}/proc -t proc
64+
sudo mount none ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}}/dev/pts -t devpts
65+
sudo mount none ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}}/sys -t sysfs
66+
echo " deb [trusted=yes] http://packages.rusoft.ru/ppa/rusoft/python ubuntu-${{matrix.os.release}} main" > rusoft-python.list
67+
echo " deb [trusted=yes] http://packages.rusoft.ru/ppa/rusoft/backports ubuntu-${{matrix.os.release}} main" > rusoft-backports.list
68+
echo " deb [trusted=yes] http://packages.rusoft.ru/ppa/rusoft/packages ubuntu-${{matrix.os.release}} main" > rusoft-packages.list
69+
echo "# deb [trusted=yes] http://ppa.launchpadcontent.net/deadsnakes/ppa/ubuntu ${{matrix.os.release}} main" > deadsnakes.list
70+
echo "# deb [trusted=yes] http://apt.llvm.org/${{matrix.os.release}}/ llvm-toolchain-${{matrix.os.release}}-20 main " > clang.list
71+
sudo chroot ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}} /usr/bin/apt update
72+
sudo cp -v ./*.list ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}}/etc/apt/sources.list.d
73+
sudo chroot ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}} /bin/bash -c 'cd /etc/apt/trusted.gpg.d && wget http://packages.rusoft.ru/apt/public.gpg -Orusoft.gpg'
74+
sudo chroot ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}} /bin/bash -c 'cd /etc/apt/trusted.gpg.d && wget http://packages.rusoft.ru/apt/public-old.gpg -Orusoft-old.gpg'
75+
sudo chroot ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}} /bin/bash -c 'apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys A2CE4BCCC50209DD || true'
76+
sudo chroot ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}} /usr/bin/apt update
77+
sudo chroot ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}} /usr/bin/apt dist-upgrade -y
78+
sudo chroot ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}} /usr/bin/apt -fy install
79+
sudo chroot ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}} /usr/bin/apt upgrade -y
80+
81+
- name: Pepare dev files
82+
if: matrix.os.matrix == 'linux'
83+
run: |
84+
sudo chroot ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}} /usr/bin/apt-cache search libzstd-dev
85+
sudo chroot ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}} /usr/bin/apt-cache madison libzstd
86+
sudo chroot ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}} /usr/bin/apt install -y libzstd-dev
87+
88+
- name: Pepare for ${{matrix.os.pypkg}} ${{matrix.os.arch}}
89+
if: matrix.os.matrix == 'linux'
90+
run: |
91+
sudo chroot ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}} /usr/bin/apt-cache madison clang-18
92+
sudo chroot ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}} /usr/bin/apt install -y ${{matrix.os.pypkg}}-dev gcc pkg-config ${{matrix.os.pypkgadd}}
93+
sudo chroot ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}} /bin/bash -c "wget ${{matrix.os.getpipurl}} -O get-pip.py && ${{matrix.os.pypkg}} get-pip.py"
94+
sudo chroot ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}} /bin/bash -c "${{matrix.os.pypkg}} -m pip install -U 'setuptools'"
95+
96+
- name: Pepare source and build wheel for ${{matrix.os.pypkg}} ${{matrix.os.arch}}
97+
if: matrix.os.matrix == 'linux'
98+
run: |
99+
sudo mkdir -p ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}}/build_wheel/
100+
version=`cat version`
101+
echo "$version"
102+
sudo tar -xvf dist/zstd-$version.tar.gz -C ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}}/build_wheel/
103+
false && sudo chroot ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}} env
104+
105+
sudo chroot ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}} pkg-config libzstd --modversion
106+
sudo chroot ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}} echo GCC-5
107+
sudo chroot ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}} gcc-5 -v
108+
sudo chroot ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}} /bin/bash -c "cd /build_wheel/zstd-$version && _CC=clang-20 ZSTD_ASM=1 ZSTD_WARNINGS=1 ZSTD_WERRORS=1 _ZSTD_SMALL=1 _ZSTD_ASM_BMI2=1 ZSTD_THREADS=1 ZSTD_EXTERNAL=1 CC=gcc-5 ${{matrix.os.pypkg}} setup.py bdist_wheel "
109+
110+
- name: Test wheel for ${{matrix.os.pypkg}} ${{matrix.os.arch}}
111+
if: matrix.os.matrix == 'linux'
112+
run: |
113+
false && sudo chroot ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}} find /build_wheel/
114+
version=`cat version`
115+
echo version=$version
116+
sudo cp -v glibc-check.sh ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}}/build_wheel/zstd-$version/
117+
sudo chroot ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}} /bin/bash -c "cd /build_wheel/zstd-$version/ ; echo LLH; ls -lh build/*/zstd.so; echo LDD; ldd build/*/zstd.so;echo FILE; file build/*/zstd.so; echo LIBC; bash ./glibc-check.sh build/*/zstd.so"
118+
sudo chroot ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}} /bin/bash -c "cd /build_wheel/zstd-$version && ${{matrix.os.pypkg}} setup.py test"
119+
120+
- name: Pepare wheel for upload
121+
if: matrix.os.matrix == 'linux'
122+
run: |
123+
version=`cat version`
124+
echo version=$version
125+
sudo cp -v ./ubuntu-${{matrix.os.release}}-${{matrix.os.arch}}/build_wheel/zstd-$version/dist/* ./dist
126+
sudo ls -lh ./dist/*
127+
sudo mv -v ./dist/"zstd-$version-${{matrix.os.pyengine_tag}}-linux_${{matrix.os.tag_arch}}.whl" ./dist/"zstd-$version-${{matrix.os.pyengine_tag}}-${{matrix.os.libc_tag}}_${{matrix.os.tag_arch}}.whl"
128+

‎README.rst‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,9 @@ python-zstd
2323
.. |cpython27x86| image:: https://github.com/sergey-dryabzhinsky/python-zstd/actions/workflows/Build_wheels_for_cpython27_x86.yml/badge.svg
2424
:target: https://github.com/sergey-dryabzhinsky/python-zstd/actions/workflows/Build_wheels_for_cpython27_x86.yml
2525

26+
.. |cpython27x86ext| image:: https://github.com/sergey-dryabzhinsky/python-zstd/actions/workflows/Build_wheels_for_cpython27_x86_ext.yml/badge.svg
27+
:target: https://github.com/sergey-dryabzhinsky/python-zstd/actions/workflows/Build_wheels_for_cpython27_x86_ext.yml
28+
2629
.. |cpython27armhf| image:: https://github.com/sergey-dryabzhinsky/python-zstd/actions/workflows/Build_wheels_for_cpython27_armhf.yml/badge.svg
2730
:target: https://github.com/sergey-dryabzhinsky/python-zstd/actions/workflows/Build_wheels_for_cpython27_armhf.yml
2831

@@ -123,6 +126,8 @@ python-zstd
123126
+============================+======================+
124127
| cpython 2.7 x86 u18 | |cpython27x86| |
125128
+----------------------------+----------------------+
129+
| cpython 2.7 x86 u18 ext | |cpython27x86ext| |
130+
+----------------------------+----------------------+
126131
| cpython 2.7 x64 u18 | |cpython27x64| |
127132
+----------------------------+----------------------+
128133
| cpython 2.7 armhf u18 | |cpython27armhf| |
@@ -496,6 +501,16 @@ ZSTD_version (): string|bytes
496501

497502
Since: 1.3.4.3
498503

504+
ZSTD_version_compiled (): string|bytes
505+
Returns ZSTD library doted version string, wi which are compiled.
506+
507+
Since: 1.5.7.3
508+
509+
ZSTD_version_loaded (): string|bytes
510+
Returns ZSTD library doted version string, with which currently loaded.
511+
512+
Since: 1.5.7.3
513+
499514
ZSTD_version_number (): int
500515
Returns ZSTD library version in format: MAJOR*100*100 + MINOR*100 + RELEASE.
501516

‎setup.py‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -273,7 +273,7 @@ def which(bin_exe):
273273
pkgconf = False
274274

275275
#if SUP_EXTERNAL:
276-
if platform.system() == "Linux" and "build_ext" in sys.argv or "build" in sys.argv or "bdist_wheel" in sys.argv:
276+
if platform.system() == "Linux" and "build_ext" in sys.argv or "build" in sys.argv or "bdist_wheel" in sys.argv or "test" in sys.argv:
277277
# You should add external library by option: --libraries zstd
278278
# And probably include paths by option: --include-dirs /usr/include/zstd
279279
# And probably library paths by option: --library-dirs /usr/lib/i386-linux-gnu
@@ -528,7 +528,7 @@ def build_extensions(self):
528528
'compress/zstd_preSplit.c',
529529
'compress/zstdmt_compress.c',
530530
'compress/zstd_fast.c',
531-
'compress/zstd_double_fast.c',
531+
# 'compress/zstd_double_fast.c',
532532
'compress/zstd_lazy.c',
533533
'compress/zstd_opt.c',
534534
'compress/zstd_ldm.c',
@@ -557,10 +557,16 @@ def build_extensions(self):
557557
]:
558558
zstdFiles.append('zstd/lib/'+f)
559559

560+
# files needed always, even for external
560561
zstdFiles.append('src/debug.c')
561562
zstdFiles.append('src/util.c')
562563
zstdFiles.append('src/python-zstd.c')
563564

565+
#for f in [
566+
# 'decompress/zstd_decompress.c',
567+
# 'decompress/zstd_ddict.c',
568+
# ]:
569+
# zstdFiles.append('zstd/lib/'+f)
564570

565571
# Another dirty hack
566572
def my_test_suite():
@@ -587,6 +593,8 @@ def my_test_suite():
587593
ld=f.read()
588594
f.close()
589595

596+
if SUP_DEBUG:
597+
print("debug: ext_libraries:%r" % (ext_libraries))
590598
setup(
591599
name='zstd',
592600
version=PKG_VERSION_STR,

‎src/python-zstd.c‎

Lines changed: 20 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -459,9 +459,24 @@ static PyObject *py_zstd_module_version(PyObject* self, PyObject *args)
459459
}
460460

461461
/**
462-
* Returns ZSTD library version as string
462+
* Returns ZSTD library version as string - compiled with
463463
*/
464-
static PyObject *py_zstd_library_version(PyObject* self, PyObject *args)
464+
static PyObject *py_zstd_library_version_compiled(PyObject* self, PyObject *args)
465+
{
466+
UNUSED(self);
467+
UNUSED(args);
468+
469+
#if PY_MAJOR_VERSION >= 3
470+
return PyUnicode_FromFormat("%s", ZSTD_VERSION_STRING);
471+
#else
472+
return PyString_FromFormat("%s", ZSTD_VERSION_STRING);
473+
#endif
474+
}
475+
476+
/**
477+
* Returns ZSTD library version as string - loaded with
478+
*/
479+
static PyObject *py_zstd_library_version_loaded(PyObject* self, PyObject *args)
465480
{
466481
UNUSED(self);
467482
UNUSED(args);
@@ -677,7 +692,9 @@ static PyMethodDef ZstdMethods[] = {
677692
{"dumps", py_zstd_compress_mt, METH_VARARGS, COMPRESS_DOCSTRING},
678693
{"loads", py_zstd_uncompress, METH_VARARGS, UNCOMPRESS_DOCSTRING},
679694
{"version", py_zstd_module_version, METH_NOARGS, VERSION_DOCSTRING},
680-
{"ZSTD_version", py_zstd_library_version, METH_NOARGS, ZSTD_VERSION_DOCSTRING},
695+
{"ZSTD_version", py_zstd_library_version_compiled, METH_NOARGS, ZSTD_VERSION_DOCSTRING},
696+
{"ZSTD_version_compiled", py_zstd_library_version_compiled, METH_NOARGS, NULL},
697+
{"ZSTD_version_loaded", py_zstd_library_version_loaded, METH_NOARGS, NULL},
681698
{"ZSTD_version_number", py_zstd_library_version_int, METH_NOARGS, ZSTD_INT_VERSION_DOCSTRING},
682699
{"ZSTD_threads_count", py_zstd_threads_count, METH_NOARGS, ZSTD_THREADS_COUNT_DOCSTRING},
683700
{"ZSTD_max_threads_count", py_zstd_max_threads_count, METH_NOARGS, ZSTD_MAX_THREADS_COUNT_DOCSTRING},

‎src/python-zstd.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -138,7 +138,8 @@ static PyObject *py_zstd_compress_mt2(PyObject* self, PyObject *args);
138138
static PyObject *py_zstd_uncompress(PyObject* self, PyObject *args);
139139
static PyObject *py_zstd_check(PyObject* self, PyObject *args);
140140
static PyObject *py_zstd_module_version(PyObject* self, PyObject *args);
141-
static PyObject *py_zstd_library_version(PyObject* self, PyObject *args);
141+
static PyObject *py_zstd_library_version_compiled(PyObject* self, PyObject *args);
142+
static PyObject *py_zstd_library_version_loaded(PyObject* self, PyObject *args);
142143
static PyObject *py_zstd_library_version_int(PyObject* self, PyObject *args);
143144
static PyObject *py_zstd_library_external(PyObject* self, PyObject *args);
144145
static PyObject *py_zstd_with_threads(PyObject* self, PyObject *args);

‎tests/base.py‎

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -22,16 +22,18 @@ def raise_skip(msg):
2222
log = logging.getLogger('ZSTD')
2323
log.info("Python version: %s" % sys.version)
2424
log.info("Machine:%s; processor:%s; system:%r; release:%r" % ( platform.machine(), platform.processor(), platform.system(), platform.release()))
25-
log.info("Module exported interface:%r"% dir(zstd))
26-
log.info("libzstd linked external:%r"% zstd.ZSTD_external())
27-
log.info("libzstd built with legacy formats support:%r"% zstd.ZSTD_legacy_support())
28-
log.info("zstd max number of threads:%r"% zstd.ZSTD_max_threads_count())
29-
log.info("zstd found CPU cores :%r"% zstd.ZSTD_threads_count())
30-
log.info("zstd default compression level:%r"% zstd.ZSTD_default_compression_level())
31-
log.info("zstd max compression level:%r"% zstd.ZSTD_max_compression_level())
32-
log.info("zstd min compression level:%r"% zstd.ZSTD_min_compression_level())
25+
log.info("Module exported interfac e:%r"% dir(zstd))
26+
log.info("libzstd linked external: %r"% zstd.ZSTD_external())
27+
log.info("libzstd built with legacy formats support: %r"% zstd.ZSTD_legacy_support())
28+
log.info("zstd max number of threads: %r"% zstd.ZSTD_max_threads_count())
29+
log.info("zstd found CPU cores : %r"% zstd.ZSTD_threads_count())
30+
log.info("zstd default compression level: %r"% zstd.ZSTD_default_compression_level())
31+
log.info("zstd max compression level: %r"% zstd.ZSTD_max_compression_level())
32+
log.info("zstd min compression level: %r"% zstd.ZSTD_min_compression_level())
3333
log.info("pyzstd module version: %r"% zstd.version())
3434
log.info("libzstd version: %r"% zstd.ZSTD_version())
35+
log.info("Compiled with libzstd version: %r"% zstd.ZSTD_version_compiled())
36+
log.info("Loaded with libzstd version: %r"% zstd.ZSTD_version_loaded())
3537

3638

3739
# Classic lorem ipsum

0 commit comments

Comments
 (0)