Merge pull request #708 from furlongm/zstd-support

add support for zstd compression in deb and rpm repos

Author: furlongm

debian/control

@@ -20,7 +20,7 @@ Depends: ${misc:Depends}, python3 (>= 3.11), python3-django (>= 4.2),
  python3-requests, python3-colorama, python3-magic, python3-humanize,
  python3-yaml, libapache2-mod-wsgi-py3, apache2, sqlite3,
  celery, python3-celery, python3-django-celery-beat, redis-server,
- python3-redis, python3-git, python3-django-taggit
+ python3-redis, python3-git, python3-django-taggit, python3-zstandard
 Suggests: python3-mysqldb, python3-psycopg2, python3-pymemcache, memcached
 Description: Django-based patch status monitoring tool for linux systems.
  .

repos/repo_types/deb.py

@@ -71,7 +71,12 @@ def refresh_deb_repo(repo):
         are and then fetches and extracts packages from those files.
     """
 
-    formats = ['Packages.xz', 'Packages.bz2', 'Packages.gz', 'Packages']
+    formats = [
+        'Packages.xz',
+        'Packages.bz2',
+        'Packages.gz',
+        'Packages',
+    ]
 
     ts = get_datetime_now()
     enabled_mirrors = repo.mirror_set.filter(refresh=True, enabled=True)

repos/repo_types/rpm.py

@@ -57,10 +57,12 @@ def refresh_rpm_repo_mirrors(repo, errata_only=False):
         which type of repo it is, then refreshes the mirrors
     """
     formats = [
+        'repodata/repomd.xml.zst',
         'repodata/repomd.xml.xz',
         'repodata/repomd.xml.bz2',
         'repodata/repomd.xml.gz',
         'repodata/repomd.xml',
+        'suse/repodata/repomd.xml.zst',
         'suse/repodata/repomd.xml.xz',
         'suse/repodata/repomd.xml.bz2',
         'suse/repodata/repomd.xml.gz',

requirements.txt

@@ -19,3 +19,4 @@ redis==6.4.0
 django-celery-beat==2.7.0
 tqdm==4.67.1
 cvss==3.4
+zstandard==0.25.0

setup.cfg

@@ -25,6 +25,7 @@ requires = /usr/bin/python3
     python3-importlib-metadata
     python3-cvss
     python3-redis
+    python3-zstandard
     redis
     celery
     python3-django-celery-beat

util/__init__.py

@@ -21,6 +21,11 @@
 import zlib
 import lzma
 import os
+try:
+    # python 3.14+ - can also remove the dependency at that stage
+    from compression import zstd
+except ImportError:
+    import zstandard as zstd
 from datetime import datetime, timezone
 from enum import Enum
 from hashlib import md5, sha1, sha256, sha512
@@ -202,6 +207,16 @@ def unxz(contents):
         error_message.send(sender=None, text='lzma: ' + e)
 
 
+def unzstd(contents):
+    """ unzstd contents in memory and return the data
+    """
+    try:
+        zstddata = zstd.ZstdDecompressor().stream_reader(contents).read()
+        return zstddata
+    except zstd.ZstdError as e:
+        error_message.send(sender=None, text='zstd: ' + e)
+
+
 def extract(data, fmt):
     """ Extract the contents based on mimetype or file ending. Return the
         unmodified data if neither mimetype nor file ending matches, otherwise
@@ -214,6 +229,8 @@ def extract(data, fmt):
         m = magic.open(magic.MAGIC_MIME)
         m.load()
         mime = m.buffer(data).split(';')[0]
+    if mime == 'application/zstd' or fmt.endswith('zst'):
+        return unzstd(data)
     if mime == 'application/x-xz' or fmt.endswith('xz'):
         return unxz(data)
     elif mime == 'application/x-bzip2' or fmt.endswith('bz2'):