Merge pull request #1020 from confluentinc/tst152

Fix packaging, tests, etc, bump to 1.6.0

Author: edenhill

.appveyor.yml

@@ -1,73 +1,106 @@
 env:
  global:
-  - LIBRDKAFKA_VERSION=v1.5.2
+  - LIBRDKAFKA_VERSION=v1.6.0
+
 jobs:
  include:
   - name: "Source package verification with Python 2.7 (Linux)"
     os: linux
     language: python
-    dist: trusty
+    dist: xenial
     python: "2.7"
     env: LD_LIBRARY_PATH="$PWD/tmp-build/lib"
+    services: docker
+
   - name: "Source package verification with Python 3.6 (Linux)"
     os: linux
     language: python
-    dist: trusty
+    dist: xenial
     python: "3.6"
     env: LD_LIBRARY_PATH="$PWD/tmp-build/lib"
+    services: docker
+
   - name: "Source package verification with Python 2.7 (OSX)"
     os: osx
     python: "2.7"
     env: DYLD_LIBRARY_PATH="$PWD/tmp-build/lib" INTERPRETER_VERSION="2.7.17"
+
   - name: "Source package verification with Python 3.6 (OSX) +docs"
     os: osx
     python: "3.6"
     env: DYLD_LIBRARY_PATH="$PWD/tmp-build/lib" MK_DOCS="y" INTERPRETER_VERSION="3.6.5"
-  - name: "cibuildwheel (OSX)"
-    os: osx
-    env: CIBW_BEFORE_BUILD="tools/bootstrap-librdkafka.sh --require-ssl ${LIBRDKAFKA_VERSION} tmp" CFLAGS="-Itmp/include" LDFLAGS="-Ltmp/lib" INTERPRETER_VERSION="2.7.17"
-  - name: "cibuildwheel (manylinux)"
-    os: linux
-    dist: trusty
-    env:
-      - CIBW_BEFORE_BUILD="tools/prepare-cibuildwheel-linux.sh ${LIBRDKAFKA_VERSION}"
-      - PYTHON_CONFIGURE_OPTS="--enable-unicode=ucs4 --with-wide-unicode"
-      - CIBW_MANYLINUX_X86_64_IMAGE="manylinux1"
-      - CIBW_MANYLINUX_I686_IMAGE="manylinux1"
+
+  - name: "Wheels: Windows x64"
+    if: tag is present
+    os: windows
+    language: shell
+    env: BUILD_WHEELS=1
+    before_install:
+      - choco install python --version 3.8.0
+      - export PATH="/c/Python38:/c/Python38/Scripts:$PATH"
+      # make sure it's on PATH as 'python3'
+      - ln -s /c/Python38/python.exe /c/Python38/python3.exe
+    install:
+      - bash tools/wheels/install-librdkafka.sh ${LIBRDKAFKA_VERSION#v} dest
+    script:
+      - tools/wheels/build-wheels.bat x64 win_amd64 dest wheelhouse
+
+  - name: "Wheels: Windows x86"
+    if: tag is present
+    os: windows
+    language: shell
+    env: BUILD_WHEELS=1
+    before_install:
+      - choco install python --version 3.8.0
+      - export PATH="/c/Python38:/c/Python38/Scripts:$PATH"
+      # make sure it's on PATH as 'python3'
+      - ln -s /c/Python38/python.exe /c/Python38/python3.exe
+    install:
+      - bash tools/wheels/install-librdkafka.sh ${LIBRDKAFKA_VERSION#v} dest
+    script:
+      - tools/wheels/build-wheels.bat x86 win32 dest wheelhouse
+
+  - name: "Wheels: Linux x64"
+    if: tag is present
     language: python
-    python: "2.7"
+    python: "3.8"
     services: docker
+    env: BUILD_WHEELS=1
+    script: tools/wheels/build-wheels.sh ${LIBRDKAFKA_VERSION#v} wheelhouse
+
+  - name: "Wheels: MacOSX x64"
+    if: tag is present
+    os: osx
+    language: shell
+    env: BUILD_WHEELS=1
+    script: tools/wheels/build-wheels.sh ${LIBRDKAFKA_VERSION#v} wheelhouse
 
-# See https://cibuildwheel.readthedocs.io/en/latest/options/ for CIBW* vars
 
 # Install test dependencies unconditionally
 # Travis OSX envs requires some setup; see tools/prepare-osx.sh
 # Install cibuildwheel if this is a tagged PR
 before_install:
-  - if [[ $TRAVIS_OS_NAME == "osx" ]]; then tools/prepare-osx.sh ${INTERPRETER_VERSION} /tmp/venv && source /tmp/venv/bin/activate; fi
-  - if [[ -n $TRAVIS_TAG && -n $CIBW_BEFORE_BUILD ]]; then pip install -U -r tools/requirements-manylinux.txt --force-reinstall ; fi
+  - if [[ $TRAVIS_OS_NAME == "osx" && $BUILD_WHEELS != 1 ]]; then tools/prepare-osx.sh ${INTERPRETER_VERSION} /tmp/venv && source /tmp/venv/bin/activate; fi
 
-# Install interceptors
-# Install librdkafka if not CIBW_BEFORE_BUILD
-# Install confluent_kafka[avro] if not CIBW_BEFORE_BUILD
 install:
+ # Install interceptors
  - tools/install-interceptors.sh
- - pip install -r tests/requirements.txt
- - pip install tox
- - flake8
+ - if [[ $BUILD_WHEELS != 1 ]]; then pip install -r tests/requirements.txt ; fi
  - if [[ $MK_DOCS == y ]]; then pip install -r docs/requirements.txt; fi
- - if [[ -z $CIBW_BEFORE_BUILD ]]; then tools/bootstrap-librdkafka.sh --require-ssl ${LIBRDKAFKA_VERSION} tmp-build && pip install --global-option=build_ext --global-option="-Itmp-build/include/" --global-option="-Ltmp-build/lib" . .[avro]; fi
+ # Install librdkafka and confluent_kafka[avro] if not building wheels
+ - if [[ $BUILD_WHEELS != 1 ]]; then pip install -U protobuf && tools/bootstrap-librdkafka.sh --require-ssl ${LIBRDKAFKA_VERSION} tmp-build ; fi
+
 
-# Build wheels
-# Make plugins available for tests
-# Execute tests if not CIBW_BEFORE_BUILD [osx, linux]
-# Execute integration tests if CIBW_BEFORE_BUILD
-# Build docs if MK_DOCS
+
+# Note: Will not be run for wheel builds.
 script:
- - if [[ -n $TRAVIS_TAG && -n $CIBW_BEFORE_BUILD ]]; then cibuildwheel --output-dir wheelhouse1 && tools/fixup-wheels.sh wheelhouse1 wheelhouse; fi
+ - flake8
+ # Build package
+ -  pip install --global-option=build_ext --global-option="-Itmp-build/include/" --global-option="-Ltmp-build/lib" . .[avro] .[schema-registry] .[json] .[protobuf]
  - ldd staging/libs/* || otool -L staging/libs/* || true
- - [[ -n $CIBW_BEFORE_BUILD ]] || LD_LIBRARY_PATH=$LD_LIBRARY_PATH:staging/libs DYLD_LIBRARY_PATH=$DYLD_LIBRARY_PATH:staging/libs tox
- - if [[ -n $TRAVIS_TAG && $TRAVIS_OS_NAME == osx && -n $CIBW_BEFORE_BUILD ]]; then tools/test-wheel.sh wheelhouse; fi
+ # Run tests
+ - if [[ $TRAVIS_OS_NAME == "linux" ]]; then LD_LIBRARY_PATH=$LD_LIBRARY_PATH:staging/libs DYLD_LIBRARY_PATH=$DYLD_LIBRARY_PATH:staging/libs python -m pytest --timeout 600 --ignore=tmp-build || travis_terminate 1; fi
+ # Build docs
  - if [[ $MK_DOCS == y ]]; then make docs; fi
 
 deploy:
@@ -82,8 +115,8 @@ deploy:
  local-dir: wheelhouse
  upload_dir: confluent-kafka-python/p-confluent-kafka-python__bld-travis__plat-${TRAVIS_OS_NAME}__tag-${TRAVIS_TAG}__sha-${TRAVIS_COMMIT}__bid-${TRAVIS_BUILD_ID}__
  acl: public_read
- skip_cleanup: true
+ cleanup: false
  on:
   repo: confluentinc/confluent-kafka-python
   tags: true
-  condition: "-n $CIBW_BEFORE_BUILD"
+  condition: "$BUILD_WHEELS == 1"

.travis.yml

@@ -57,7 +57,7 @@
 # built documents.
 #
 # The short X.Y version.
-version = '1.5.2'
+version = '1.6.0'
 # The full version, including alpha/beta/rc tags.
 release = version
 

docs/conf.py

@@ -30,7 +30,7 @@ FROM alpine:3.12
 
 COPY . /usr/src/confluent-kafka-python
 
-ENV LIBRDKAFKA_VERSION v1.5.2
+ENV LIBRDKAFKA_VERSION v1.6.0
 ENV KAFKACAT_VERSION master
 
 

examples/docker/Dockerfile.alpine

@@ -3,5 +3,7 @@ fastapi
 pydantic
 uvicorn
 six
+pyrsistent==0.16.1;python_version<"3.0"
+pyrsistent;python_version>"3.0"
 jsonschema
 protobuf

examples/requirements.txt

@@ -31,7 +31,9 @@
                  'avro-python3==1.10.0;python_version>"3.0"'
                  ] + SCHEMA_REGISTRY_REQUIRES
 
-JSON_REQUIRES = ['jsonschema'] + SCHEMA_REGISTRY_REQUIRES
+JSON_REQUIRES = ['pyrsistent==0.16.1;python_version<"3.0"',
+                 'pyrsistent;python_version>"3.0"',
+                 'jsonschema'] + SCHEMA_REGISTRY_REQUIRES
 
 PROTO_REQUIRES = ['protobuf'] + SCHEMA_REGISTRY_REQUIRES
 
@@ -74,7 +76,7 @@ def get_install_requirements(path):
 setup(name='confluent-kafka',
       # Make sure to bump CFL_VERSION* in confluent_kafka/src/confluent_kafka.h
       # and version and release in docs/conf.py.
-      version='1.5.2',
+      version='1.6.0',
       description='Confluent\'s Python client for Apache Kafka',
       author='Confluent Inc',
       author_email='[email protected]',

setup.py

@@ -1,4 +1,6 @@
 fastavro>=0.23.0
+pyrsistent==0.16.1;python_version<"3.0"
+pyrsistent;python_version>"3.0"
 jsonschema
 protobuf
 requests

src/confluent_kafka/schema_registry/requirements.txt

@@ -42,28 +42,28 @@
  *  0xMMmmRRPP
  *  MM=major, mm=minor, RR=revision, PP=patchlevel (not used)
  */
-#define CFL_VERSION     0x01050200
-#define CFL_VERSION_STR "1.5.2"
+#define CFL_VERSION     0x01060000
+#define CFL_VERSION_STR "1.6.0"
 
 /**
  * Minimum required librdkafka version. This is checked both during
  * build-time (just below) and runtime (see confluent_kafka.c).
  * Make sure to keep the MIN_RD_KAFKA_VERSION, MIN_VER_ERRSTR and #error
  * defines and strings in sync.
  */
-#define MIN_RD_KAFKA_VERSION 0x01040000
+#define MIN_RD_KAFKA_VERSION 0x01060000
 
 #ifdef __APPLE__
-#define MIN_VER_ERRSTR "confluent-kafka-python requires librdkafka v1.4.0 or later. Install the latest version of librdkafka from Homebrew by running `brew install librdkafka` or `brew upgrade librdkafka`"
+#define MIN_VER_ERRSTR "confluent-kafka-python requires librdkafka v1.6.0 or later. Install the latest version of librdkafka from Homebrew by running `brew install librdkafka` or `brew upgrade librdkafka`"
 #else
-#define MIN_VER_ERRSTR "confluent-kafka-python requires librdkafka v1.4.0 or later. Install the latest version of librdkafka from the Confluent repositories, see http://docs.confluent.io/current/installation.html"
+#define MIN_VER_ERRSTR "confluent-kafka-python requires librdkafka v1.6.0 or later. Install the latest version of librdkafka from the Confluent repositories, see http://docs.confluent.io/current/installation.html"
 #endif
 
 #if RD_KAFKA_VERSION < MIN_RD_KAFKA_VERSION
 #ifdef __APPLE__
-#error "confluent-kafka-python requires librdkafka v1.4.0 or later. Install the latest version of librdkafka from Homebrew by running `brew install librdkafka` or `brew upgrade librdkafka`"
+#error "confluent-kafka-python requires librdkafka v1.6.0 or later. Install the latest version of librdkafka from Homebrew by running `brew install librdkafka` or `brew upgrade librdkafka`"
 #else
-#error "confluent-kafka-python requires librdkafka v1.4.0 or later. Install the latest version of librdkafka from the Confluent repositories, see http://docs.confluent.io/current/installation.html"
+#error "confluent-kafka-python requires librdkafka v1.6.0 or later. Install the latest version of librdkafka from the Confluent repositories, see http://docs.confluent.io/current/installation.html"
 #endif
 #endif
 

src/confluent_kafka/src/confluent_kafka.h

@@ -5,7 +5,7 @@ TARGET_DIR=$(WORK_DIR)/../gen
 
 PROTOS := common_proto.proto DependencyTestProto.proto exampleProtoCriteo.proto $\
          metadata_proto.proto NestedTestProto.proto PublicTestProto.proto $\
-         Sint32Value.proto Sint64Value.proto TestProto.proto
+         SInt32Value.proto SInt64Value.proto TestProto.proto
 
 compile: $(PROTOS)
 	for proto in $(PROTOS); do \

tests/integration/schema_registry/data/Makefile

@@ -16,7 +16,7 @@
 #
 import pytest
 
-from confluent_kafka import TopicPartition, KafkaException
+from confluent_kafka import TopicPartition, KafkaException, KafkaError
 from confluent_kafka.error import ConsumeError
 from confluent_kafka.schema_registry.protobuf import ProtobufSerializer, ProtobufDeserializer
 from .gen import metadata_proto_pb2
@@ -145,6 +145,6 @@ def dr(err, msg):
                      partition=0)
     producer.flush()
 
-    with pytest.raises(ConsumeError,
-                       match="Error parsing message"):
+    with pytest.raises(ConsumeError) as e:
         consumer.poll()
+    assert e.value.code == KafkaError._KEY_DESERIALIZATION