Update libbrotli to 1.0.9

Author: sethmlarson

.github/workflows/ci.yml

@@ -32,6 +32,7 @@ jobs:
         with:
           submodules: recursive
       - uses: actions/setup-python@v1
+        if: runner.os != 'Windows'  # see issue #174
         with:
           python-version: 2.7
       - uses: actions/setup-python@v1
@@ -40,10 +41,16 @@ jobs:
       - uses: actions/setup-python@v1
         with:
           python-version: 3.9
-      - name: Install Visual C++ for Python 2.7
+      # - name: Install Visual C++ for Python 2.7
+      #   if: runner.os == 'Windows'
+      #   run: choco install vcpython27 -f -y
+      - name: Run tests on Windows  # see issue #174
         if: runner.os == 'Windows'
-        run: choco install vcpython27 -f -y
-      - name: Run tests
+        run: |
+          python -m pip install tox
+          tox -e py35,py39
+      - name: Run tests on Linux and macOS
+        if: runner.os != 'Windows'
         run: |
           python -m pip install tox
           tox --skip-missing-interpreters
@@ -85,10 +92,16 @@ jobs:
 
       - name: Install cibuildwheel
         run: python -m pip install cibuildwheel
-      - name: Install Visual C++ for Python 2.7
+      # - name: Install Visual C++ for Python 2.7
+      #   if: runner.os == 'Windows'
+      #   run: choco install vcpython27 -f -y
+      - name: Build wheels for Windows  # see issue #174
         if: runner.os == 'Windows'
-        run: choco install vcpython27 -f -y
-      - name: Build wheels
+        run: python -m cibuildwheel --output-dir wheelhouse
+        env:
+          CIBW_BUILD: cp35-${{ matrix.name }}* pp*-${{ matrix.name }}*
+      - name: Build wheels for Linux and macOS
+        if: runner.os != 'Windows'
         run: python -m cibuildwheel --output-dir wheelhouse
         env:
           CIBW_BUILD: cp27-${{ matrix.name }}* cp35-${{ matrix.name }}* pp*-${{ matrix.name }}*
@@ -105,19 +118,19 @@ jobs:
       - uses: actions/download-artifact@v1
         with:
           name: sdist
-          path: dists/
+          path: dist/
       - uses: actions/download-artifact@v1
         with:
           name: wheels-win
-          path: dists/
+          path: dist/
       - uses: actions/download-artifact@v1
         with:
           name: wheels-macos
-          path: dists/
+          path: dist/
       - uses: actions/download-artifact@v1
         with:
           name: wheels-manylinux
-          path: dists/
+          path: dist/
       - name: Publish to PyPI
         if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/')
         uses: pypa/gh-action-pypi-publish@master

MANIFEST.in

@@ -5,10 +5,10 @@ include LICENSE README.rst HISTORY.rst
 include libbrotli/LICENSE
 
 # Include the needed .c and .h files
-recursive-include libbrotli/enc *.c *.h *.cc
-recursive-include libbrotli/dec *.c *.h *.cc
-recursive-include libbrotli/common *.c *.h *.cc
-recursive-include libbrotli/include *.c *.h *.cc
+recursive-include libbrotli/c/enc *.c *.h *.cc
+recursive-include libbrotli/c/dec *.c *.h *.cc
+recursive-include libbrotli/c/common *.c *.h *.cc
+recursive-include libbrotli/c/include *.c *.h *.cc
 
 # Include test files
 recursive-include test/ *.py

libbrotli

@@ -1,14 +1,20 @@
 #!/usr/bin/env python
 import os
+import re
 import platform
 import sys
 from setuptools import find_packages, setup
 from setuptools.command.build_ext import build_ext
 
+base_dir = os.path.dirname(os.path.abspath(__file__))
 long_description = (
     open("README.rst").read() + '\n\n' + open("HISTORY.rst").read()
 )
 
+with open(os.path.join(base_dir, "src", "brotlicffi", "__init__.py")) as f:
+    __version__ = re.search(r"__version__ = \"([^\"]+)\"", f.read()).group(1)
+
+
 class BuildClibBeforeExt(build_ext):
     """ Setuptools `develop` command (used by `pip install -e .`) only calls
     `build_ext`, unlike `install` which calls `build` and all its related
@@ -21,44 +27,43 @@ class BuildClibBeforeExt(build_ext):
     https://github.com/pypa/pip/issues/4523
     """
 
+    def get_source_files(self):
+        filenames = build_ext.get_source_files(self)
+        filenames.extend(depends)
+        return filenames
+
     def run(self):
         self.run_command("build_clib")
         build_ext.run(self)
 
+
+depends = []
 libraries = []
 USE_SHARED_BROTLI = os.environ.get("USE_SHARED_BROTLI")
 if USE_SHARED_BROTLI != "1":
+    sources = []
+
+    for root, _, filenames in os.walk("libbrotli/c"):
+        root_parts = os.path.split(root)
+        if "fuzz" in root_parts or "tools" in root_parts:
+            continue
+        for filename in filenames:
+            relpath = os.path.relpath(os.path.join(root, filename), base_dir)
+            if filename.endswith(".h"):
+                depends.append(relpath)
+            elif filename.endswith(".c"):
+                sources.append(relpath)
+
     libraries = [
         ("libbrotli", {
             "include_dirs": [
-                "libbrotli/include",
-                "libbrotli/",
+                "libbrotli/c/include",
+                "libbrotli/c/common",
+                "libbrotli/c",
                 "src/brotlicffi"
             ],
-            "sources": [
-                'libbrotli/common/dictionary.c',
-                'libbrotli/dec/huffman.c',
-                'libbrotli/dec/bit_reader.c',
-                'libbrotli/dec/decode.c',
-                'libbrotli/dec/state.c',
-                'libbrotli/enc/backward_references.c',
-                'libbrotli/enc/backward_references_hq.c',
-                'libbrotli/enc/bit_cost.c',
-                'libbrotli/enc/block_splitter.c',
-                'libbrotli/enc/histogram.c',
-                'libbrotli/enc/memory.c',
-                'libbrotli/enc/literal_cost.c',
-                'libbrotli/enc/brotli_bit_stream.c',
-                'libbrotli/enc/compress_fragment_two_pass.c',
-                'libbrotli/enc/compress_fragment.c',
-                'libbrotli/enc/cluster.c',
-                'libbrotli/enc/utf8_util.c',
-                'libbrotli/enc/encode.c',
-                'libbrotli/enc/metablock.c',
-                'libbrotli/enc/static_dict.c',
-                'libbrotli/enc/dictionary_hash.c',
-                'libbrotli/enc/entropy_encode.c'
-            ]
+            "depends": depends,
+            "sources": sources,
         }),
     ]
 
@@ -77,7 +82,7 @@ def finalize_options(self):
 
 setup(
     name="brotlicffi",
-    version="0.8.0",
+    version=__version__,
 
     description="Python CFFI bindings to the Brotli library",
     long_description=long_description,

setup.py

@@ -5,4 +5,4 @@
     Compressor, MODE_GENERIC, MODE_TEXT, MODE_FONT, error, Error
 )
 
-__version__ = "0.8.0"
+__version__ = "1.0.9.0"

src/brotlicffi/__init__.py

@@ -95,8 +95,7 @@ def compress(data,
              mode=DEFAULT_MODE,
              quality=lib.BROTLI_DEFAULT_QUALITY,
              lgwin=lib.BROTLI_DEFAULT_WINDOW,
-             lgblock=0,
-             dictionary=b''):
+             lgblock=0):
     """
     Compress a string using Brotli.
 
@@ -124,11 +123,6 @@ def compress(data,
         based on ``quality``.
     :type lgblock: ``int``
 
-    :param dictionary: A pre-set dictionary for LZ77. Please use this with
-        caution: if a dictionary is used for compression, the same dictionary
-        **must** be used for decompression!
-    :type dictionary: ``bytes``
-
     :returns: The compressed bytestring.
     :rtype: ``bytes``
     """
@@ -141,8 +135,7 @@ def compress(data,
         mode=mode,
         quality=quality,
         lgwin=lgwin,
-        lgblock=lgblock,
-        dictionary=dictionary
+        lgblock=lgblock
     )
     compressed_data = compressor._compress(data, lib.BROTLI_OPERATION_FINISH)
     assert lib.BrotliEncoderIsFinished(compressor._encoder) == lib.BROTLI_TRUE
@@ -255,8 +248,7 @@ def __init__(self,
                  mode=DEFAULT_MODE,
                  quality=lib.BROTLI_DEFAULT_QUALITY,
                  lgwin=lib.BROTLI_DEFAULT_WINDOW,
-                 lgblock=0,
-                 dictionary=b''):
+                 lgblock=0):
         enc = lib.BrotliEncoderCreateInstance(
             ffi.NULL, ffi.NULL, ffi.NULL
         )
@@ -271,13 +263,6 @@ def __init__(self,
         _set_parameter(enc, lib.BROTLI_PARAM_LGWIN, "lgwin", lgwin)
         _set_parameter(enc, lib.BROTLI_PARAM_LGBLOCK, "lgblock", lgblock)
 
-        if dictionary:
-            self._dictionary = ffi.new("uint8_t []", dictionary)
-            self._dictionary_size = len(dictionary)
-            lib.BrotliEncoderSetCustomDictionary(
-                enc, self._dictionary_size, self._dictionary
-            )
-
         self._encoder = enc
 
     def _compress(self, data, operation):

src/brotlicffi/_api.py

@@ -21,7 +21,7 @@
        #include <brotli/encode.h>
     """,
     libraries=libraries,
-    include_dirs=["libbrotli", "libbrotli/include"]
+    include_dirs=["libbrotli/c", "libbrotli/c/include", "libbrotli/c/common"]
 )
 
 ffi.cdef("""
@@ -93,20 +93,6 @@
                                                       uint8_t** next_out,
                                                       size_t* total_out);
 
-    /* Fills the new state with a dictionary for LZ77, warming up the
-       ringbuffer, e.g. for custom static dictionaries for data formats.
-       Not to be confused with the built-in transformable dictionary of Brotli.
-       |size| should be less or equal to 2^24 (16MiB), otherwise the dictionary
-       will be ignored. The dictionary must exist in memory until decoding is
-       done and is owned by the caller. To use:
-        1) Allocate and initialize state with BrotliCreateInstance
-        2) Use BrotliSetCustomDictionary
-        3) Use BrotliDecompressStream
-        4) Clean up and free state with BrotliDestroyState
-    */
-    void BrotliDecoderSetCustomDictionary(
-        BrotliDecoderState* s, size_t size, const uint8_t* dict);
-
     /* Returns true, if decoder has some unconsumed output.
        Otherwise returns false. */
     BROTLI_BOOL BrotliDecoderHasMoreOutput(const BrotliDecoderState* s);
@@ -205,15 +191,6 @@
                                           BrotliEncoderParameter p,
                                           uint32_t value);
 
-    /* Fills the new state with a dictionary for LZ77, warming up the
-       ringbuffer, e.g. for custom static dictionaries for data formats.
-       Not to be confused with the built-in transformable dictionary of Brotli.
-       To decode, use BrotliSetCustomDictionary() of the decoder with the same
-       dictionary. */
-    void BrotliEncoderSetCustomDictionary(BrotliEncoderState* state,
-                                          size_t size,
-                                          const uint8_t* dict);
-
     /* Check if encoder is in "finished" state, i.e. no more input is
        acceptable and no more output will be produced.
        Works only with BrotliEncoderCompressStream workflow.

src/brotlicffi/_build.py

@@ -5,6 +5,8 @@
 
 Tests for names that exist purely for compatibility purposes.
 """
+import re
+from os.path import abspath, join, dirname
 import brotlicffi
 
 
@@ -16,3 +18,26 @@ def test_compatible_names():
     assert brotlicffi.MODE_GENERIC is brotlicffi.BrotliEncoderMode.GENERIC
     assert brotlicffi.MODE_TEXT is brotlicffi.BrotliEncoderMode.TEXT
     assert brotlicffi.MODE_FONT is brotlicffi.BrotliEncoderMode.FONT
+
+
+def test_brotli_version():
+    """
+    Test that the __version__ starts with the
+    Brotli version that's compiled with.
+    """
+    version_h = join(
+        dirname(dirname(abspath(__file__))), "libbrotli/c/common/version.h"
+    )
+    with open(version_h) as f:
+        brotli_version = int(
+            re.search(
+                r"#define BROTLI_VERSION 0x([A-Fa-f0-9]+)", f.read()
+            ).group(1),
+            16,
+        )
+        major = brotli_version >> 24
+        minor = (brotli_version >> 12) & 0xFFF
+        patch = brotli_version & 0xFFF
+        assert brotlicffi.__version__.startswith(
+            "%d.%d.%d." % (major, minor, patch)
+        )

test/test_compatibility.py

@@ -110,14 +110,6 @@ def test_compressed_data_roundtrips(s):
     assert brotlicffi.decompress(brotlicffi.compress(s)) == s
 
 
-@given(binary(), binary())
-def test_compressed_data_with_dictionaries(s, dictionary):
-    d = brotlicffi.Decompressor(dictionary)
-    compressed = brotlicffi.compress(s, dictionary=dictionary)
-    uncompressed = d.decompress(compressed)
-    assert uncompressed == s
-
-
 @given(binary())
 def test_process_alias(s):
     c1 = brotlicffi.Compressor()