add zst file handling for Python 3.14 and below (#428)

* add zst file handling for Python 3.14 and below

* condition reordering

Author: shuchenliu

biothings/utils/common.py

@@ -173,12 +173,31 @@ def anyfile(infile, mode="r"):
         rawfile = os.path.splitext(infile)[0]
     filetype = os.path.splitext(infile)[1].lower()
 
+    # check if lower version zst handling is needed
+    lower_version_zst = False
+    if sys.version_info < (3, 14) and filetype == ".zst":
+        import zstandard as zstd
+        lower_version_zst = True
+
+    # tarfile handling. works for zst in Python >= 3.14
+    if lower_version_zst or tarfile.is_tarfile(infile):
+        if lower_version_zst:
+            f = open(infile, "rb")
+            dctx = zstd.ZstdDecompressor()
+            reader = dctx.stream_reader(f)
+            tar_file = tarfile.open(fileobj=reader, mode="r|")  # streaming mode
+        else:
+            tar_file = tarfile.open(infile, mode)
 
-    # use tarfile built-in method to check for tar file before anything else
-    if tarfile.is_tarfile(infile):
-        tar_file = tarfile.open(infile, mode)
+        extracted = None
         try:
-            extracted = tar_file.extractfile(rawfile)
+            if lower_version_zst:
+                for member in tar_file:
+                    if member.name == rawfile:
+                        extracted = tar_file.extractfile(member)
+                        break
+            else:
+                extracted = tar_file.extractfile(rawfile)
         except KeyError:
             # provided rawfile does not appear in the tarball
             tar_file.close()
@@ -189,6 +208,9 @@ def anyfile(infile, mode="r"):
             tar_file.close()
             raise Exception("invalid target file: must be a regular file or a link")
 
+        if lower_version_zst:
+            return extracted
+
         return io.TextIOWrapper(extracted)
 
     if filetype == ".gz":

pyproject.toml

@@ -71,6 +71,7 @@ dependencies = [
     "PyYAML>=5.1",
     'orjson>=3.10.16; python_version < "3.14.0"',  # a faster json lib supports inf/nan and datetime, v3.10.16 is the first version requires Python 3.9+
     'orjson==3.11.4; python_version >= "3.14.0"',  # orjson 3.11.5 cannot be built on Python 3.14t for now
+    'zstandard>=0.21.0; python_version<"3.14"', # we need zst library before 3.14
 ]
 
 [project.optional-dependencies]