Changes related to release (#1504)

pranavrth · web-flow · commit fa1aa03b41ce · 2023-01-22T23:56:24.000+05:30
* Installing python before running cibuildwheel for macos
* Upgraded cibuildwheel version 2.12.0
* Updated delvewheel versions to 1.1.14
* Added few Changelogs. Reordered Changelogs. Updated deprecation information.
diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml
@@ -6,7 +6,7 @@ agent:
 global_job_config:
   env_vars:
     - name: LIBRDKAFKA_VERSION
-      value: v2.0.0-RC3
+      value: v2.0.2
   prologue:
     commands:
       - export HOME=$WORKSPACE
diff --git a/.travis.yml b/.travis.yml
@@ -1,7 +1,7 @@
 env:
  global:
-  - LIBRDKAFKA_VERSION=v2.0.0-RC3
-  - LIBRDKAFKA_SRC_VERSION=master
+  - LIBRDKAFKA_VERSION=v2.0.2
+  - LIBRDKAFKA_SRC_VERSION=v2.0.2
 
 jobs:
  include:
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,14 +1,10 @@
 # Confluent's Python client for Apache Kafka
 
 
-## Next Version
+## v2.0.2
+
+v2.0.2 is a feature release with the following features, fixes and enhancements:
 
- - Added metadata to `TopicPartition` type and `commit()` (#1410).
- - Added `consumer.memberid()` for getting member id assigned to
-   the consumer in a consumer group (#1154).
- - Implemented `nb_bool` method for the Producer, so that the default (which uses len)
-   will not be used. This avoids situations where producers with no enqueued items would
-   evaluate to False (#1445).
  - Added Python 3.11 wheels.
  - [KIP-222](https://cwiki.apache.org/confluence/display/KAFKA/KIP-222+-+Add+Consumer+Group+operations+to+Admin+API)
    Add Consumer Group operations to Admin API.
@@ -22,12 +18,24 @@
    - `delete_consumer_groups` Admin operation. Supports multiple groups.
    - `list_consumer_group_offsets` Admin operation. Currently, only supports 1 group with multiple partitions. Supports require_stable option.
    - `alter_consumer_group_offsets` Admin operation. Currently, only supports 1 group with multiple offsets.
- - Added `normalize.schemas` configuration property to Schema Registry client
-
-confluent-kafka-python is based on librdkafka v2.0.0, see the
-[librdkafka release notes](https://github.com/edenhill/librdkafka/releases/tag/v2.0.0)
+ - Added `normalize.schemas` configuration property to Schema Registry client (@rayokota, #1406)
+ - Added metadata to `TopicPartition` type and `commit()` (#1410).
+ - Added `consumer.memberid()` for getting member id assigned to
+   the consumer in a consumer group (#1154).
+ - Implemented `nb_bool` method for the Producer, so that the default (which uses len)
+   will not be used. This avoids situations where producers with no enqueued items would
+   evaluate to False (@vladz-sternum, #1445).
+ - Deprecated `AvroProducer` and `AvroConsumer`. Use `AvroSerializer` and `AvroDeserializer` instead.
+ - Deprecated `list_groups`. Use `list_consumer_groups` and `describe_consumer_groups` instead.
+ - Improved Consumer Example to show atleast once semantics.
+ - Improved Serialization and Deserialization Examples.
+ - Documentation Improvements.
+
+confluent-kafka-python is based on librdkafka v2.0.2, see the
+[librdkafka release notes](https://github.com/edenhill/librdkafka/releases/tag/v2.0.2)
 for a complete list of changes, enhancements, fixes and upgrade considerations.
 
+**Note: There were no v2.0.0 and v2.0.1 releases.**
 
 ## v1.9.2
 
diff --git a/docs/conf.py b/docs/conf.py
@@ -27,14 +27,14 @@
 ######################################################################
 # General information about the project.
 project = u'confluent-kafka'
-copyright = u'2016-2021, Confluent Inc.'
+copyright = u'2016-2023, Confluent Inc.'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
 # built documents.
 #
 # The short X.Y version.
-version = '1.9.2'
+version = '2.0.2'
 # The full version, including alpha/beta/rc tags.
 release = version
 ######################################################################
diff --git a/examples/docker/Dockerfile.alpine b/examples/docker/Dockerfile.alpine
@@ -30,7 +30,7 @@ FROM alpine:3.12
 
 COPY . /usr/src/confluent-kafka-python
 
-ENV LIBRDKAFKA_VERSION v1.9.2
+ENV LIBRDKAFKA_VERSION v2.0.2
 ENV KAFKACAT_VERSION master
 
 
diff --git a/setup.py b/setup.py
@@ -75,7 +75,7 @@ def get_install_requirements(path):
 setup(name='confluent-kafka',
       # Make sure to bump CFL_VERSION* in confluent_kafka/src/confluent_kafka.h
       # and version in docs/conf.py.
-      version='1.9.2',
+      version='2.0.2',
       description='Confluent\'s Python client for Apache Kafka',
       author='Confluent Inc',
       author_email='support@confluent.io',
diff --git a/src/confluent_kafka/avro/__init__.py b/src/confluent_kafka/avro/__init__.py
@@ -33,7 +33,7 @@
 
 class AvroProducer(Producer):
     """
-        .. deprecated:: 1.9.3
+        .. deprecated:: 2.0.2
 
         This class will be removed in a future version of the library.
 
@@ -119,7 +119,7 @@ def produce(self, **kwargs):
 
 class AvroConsumer(Consumer):
     """
-    .. deprecated:: 1.9.3
+    .. deprecated:: 2.0.2
 
     This class will be removed in a future version of the library.
 
diff --git a/src/confluent_kafka/src/Metadata.c b/src/confluent_kafka/src/Metadata.c
@@ -630,6 +630,9 @@ list_groups (Handle *self, PyObject *args, PyObject *kwargs) {
 }
 
 const char list_groups_doc[] = PyDoc_STR(
+        ".. deprecated:: 2.0.2"
+        "   Use :func:`list_consumer_groups` and `describe_consumer_groups` instead."
+        "\n"
         ".. py:function:: list_groups([group=None], [timeout=-1])\n"
         "\n"
         " Request Group Metadata from cluster.\n"
diff --git a/src/confluent_kafka/src/confluent_kafka.h b/src/confluent_kafka/src/confluent_kafka.h
@@ -42,28 +42,28 @@
  *  0xMMmmRRPP
  *  MM=major, mm=minor, RR=revision, PP=patchlevel (not used)
  */
-#define CFL_VERSION     0x01090200
-#define CFL_VERSION_STR "1.9.2"
+#define CFL_VERSION     0x02000200
+#define CFL_VERSION_STR "2.0.2"
 
 /**
  * Minimum required librdkafka version. This is checked both during
  * build-time (just below) and runtime (see confluent_kafka.c).
  * Make sure to keep the MIN_RD_KAFKA_VERSION, MIN_VER_ERRSTR and #error
  * defines and strings in sync.
  */
-#define MIN_RD_KAFKA_VERSION 0x010900ff
+#define MIN_RD_KAFKA_VERSION 0x020002ff
 
 #ifdef __APPLE__
-#define MIN_VER_ERRSTR "confluent-kafka-python requires librdkafka v1.9.0 or later. Install the latest version of librdkafka from Homebrew by running `brew install librdkafka` or `brew upgrade librdkafka`"
+#define MIN_VER_ERRSTR "confluent-kafka-python requires librdkafka v2.0.2 or later. Install the latest version of librdkafka from Homebrew by running `brew install librdkafka` or `brew upgrade librdkafka`"
 #else
-#define MIN_VER_ERRSTR "confluent-kafka-python requires librdkafka v1.9.0 or later. Install the latest version of librdkafka from the Confluent repositories, see http://docs.confluent.io/current/installation.html"
+#define MIN_VER_ERRSTR "confluent-kafka-python requires librdkafka v2.0.2 or later. Install the latest version of librdkafka from the Confluent repositories, see http://docs.confluent.io/current/installation.html"
 #endif
 
 #if RD_KAFKA_VERSION < MIN_RD_KAFKA_VERSION
 #ifdef __APPLE__
-#error "confluent-kafka-python requires librdkafka v1.9.0 or later. Install the latest version of librdkafka from Homebrew by running `brew install librdkafka` or `brew upgrade librdkafka`"
+#error "confluent-kafka-python requires librdkafka v2.0.2 or later. Install the latest version of librdkafka from Homebrew by running `brew install librdkafka` or `brew upgrade librdkafka`"
 #else
-#error "confluent-kafka-python requires librdkafka v1.9.0 or later. Install the latest version of librdkafka from the Confluent repositories, see http://docs.confluent.io/current/installation.html"
+#error "confluent-kafka-python requires librdkafka v2.0.2 or later. Install the latest version of librdkafka from the Confluent repositories, see http://docs.confluent.io/current/installation.html"
 #endif
 #endif
 
diff --git a/tools/RELEASE.md b/tools/RELEASE.md
@@ -116,6 +116,7 @@ tag (e.g., v0.11.4-RC5).
 
 Change to the latest librdkafka version in the following files:
 
+ * `.semaphore/semaphore.yml`
  * `.travis.yml`
  * `examples/docker/Dockerfile.alpine`
 
diff --git a/tools/wheels/build-wheels.bat b/tools/wheels/build-wheels.bat
@@ -14,7 +14,7 @@ if [%WHEELHOUSE%]==[] goto usage
 echo on
 
 set CIBW_BUILD=cp36-%BW_ARCH% cp37-%BW_ARCH% cp38-%BW_ARCH% cp39-%BW_ARCH% cp310-%BW_ARCH% cp311-%BW_ARCH%
-set CIBW_BEFORE_BUILD=python -m pip install delvewheel==0.0.6
+set CIBW_BEFORE_BUILD=python -m pip install delvewheel==1.1.4
 set CIBW_TEST_REQUIRES=-r tests/requirements.txt
 set CIBW_TEST_COMMAND=pytest {project}\tests\test_Producer.py
 rem set CIBW_BUILD_VERBOSITY=3
@@ -25,7 +25,7 @@ set CIBW_REPAIR_WHEEL_COMMAND=python -m delvewheel repair --add-path %DLL_DIR% -
 
 set PATH=%PATH%;c:\Program Files\Git\bin\
 
-python -m pip install cibuildwheel==2.11.2 || goto :error
+python -m pip install cibuildwheel==2.12.0 || goto :error
 
 python -m cibuildwheel --output-dir %WHEELHOUSE% --platform windows || goto :error
 
@@ -34,7 +34,7 @@ dir %WHEELHOUSE%
 goto :eof
 
 :usage
-@echo "Usage: %0 x86|x64 win32|win_amd64 wheelhouse-dir"
+@echo "Usage: %0 x86|x64 win32|win_amd64 librdkafka-dir wheelhouse-dir"
 exit /B 1
 
 :error
diff --git a/tools/wheels/build-wheels.sh b/tools/wheels/build-wheels.sh
@@ -16,6 +16,7 @@ export CIBW_TEST_COMMAND="pytest {project}/tests/test_Producer.py"
 
 librdkafka_version=$1
 wheeldir=$2
+cibuildwheel_version="2.12.0"
 
 if [[ -z $wheeldir ]]; then
     echo "Usage: $0 <librdkafka-nuget-version> <wheeldir>"
@@ -49,10 +50,9 @@ case $OSTYPE in
         ;;
 esac
 
-
 $this_dir/install-librdkafka.sh $librdkafka_version dest
 
-install_pkgs=cibuildwheel==2.11.2
+install_pkgs=cibuildwheel==$cibuildwheel_version
 
 python3 -m pip install ${PIP_INSTALL_OPTS} $install_pkgs ||
     pip3 install ${PIP_INSTALL_OPTS} $install_pkgs
@@ -61,6 +61,10 @@ if [[ -z $TRAVIS ]]; then
     cibw_args="--platform $os"
 fi
 
+if [[ $os == "macos" ]]; then
+    python3 $this_dir/install-macos-python-required-by-cibuildwheel.py $cibuildwheel_version
+fi
+
 LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$PWD/$lib_dir python3 -m cibuildwheel --output-dir $wheeldir $cibw_args
 
 ls $wheeldir
diff --git a/tools/wheels/install-macos-python-required-by-cibuildwheel.py b/tools/wheels/install-macos-python-required-by-cibuildwheel.py
@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+#
+#
+# Get python versions required for cibuildwheel from their config and
+# install them. This implementation is based on cibuildwheel 2.12.0
+# version. Might need tweak if something changes in cibuildwheel.
+#
+# This was added as there is a permission issue when cibuildwheel
+# tries to install these versions on its own.
+#
+
+import platform
+import sys
+import os
+import tomli
+import urllib.request
+import re
+import shutil
+
+
+cibuildwheel_version = sys.argv[1]
+config_url = f"https://raw.githubusercontent.com/pypa/cibuildwheel/v{cibuildwheel_version}/cibuildwheel/resources/build-platforms.toml"
+print(f"Config URL is '{config_url}'")
+
+response = urllib.request.urlopen(config_url).read()
+
+content = response.decode('utf-8')
+d = tomli.loads(content)
+macos_config = d['macos']['python_configurations']
+
+machine_arc = platform.machine()
+print(f"Machine Architecture is '{machine_arc}'")
+machine_arc_regex_string = f".*{machine_arc}"
+machine_arc_regex = re.compile(machine_arc_regex_string)
+
+skip_versions = os.environ['CIBW_SKIP']
+print(f"Versions to skip are '{skip_versions}'")
+skip_versions_list = skip_versions.split()
+skip_versions_regex_string = ("|".join(skip_versions_list)).replace("*", ".*")
+skip_versions_regex = re.compile(skip_versions_regex_string)
+
+py_versions_info = []
+
+for py_version_config in macos_config:
+    identifier = py_version_config['identifier']
+    if not skip_versions_regex.match(identifier) and machine_arc_regex.match(identifier):
+        pkg_url = py_version_config['url']
+        py_versions_info.append((identifier, pkg_url))
+
+tmp_download_dir = "tmp_download_dir"
+tmp_pkg_file_name = "Package.pkg"
+this_file_path = os.getcwd()
+print(f"CWD is: '{this_file_path}'")
+tmp_download_dir_full_path = os.path.join(os.getcwd(), tmp_download_dir)
+tmp_pkg_file_full_path = os.path.join(tmp_download_dir_full_path, tmp_pkg_file_name)
+if os.path.exists(tmp_download_dir_full_path):
+    shutil.rmtree(tmp_download_dir_full_path)
+os.mkdir(tmp_download_dir)
+os.chdir(tmp_download_dir)
+install_command = f"sudo installer -pkg {tmp_pkg_file_name} -target /"
+
+for py_version_info in py_versions_info:
+    identifier = py_version_info[0]
+    pkg_url = py_version_info[1]
+    print(f"Installing '{identifier}' from '{pkg_url}'")
+    os.system(f"curl {pkg_url} --output {tmp_pkg_file_name}")
+    os.system(install_command)
+    os.remove(tmp_pkg_file_full_path)
+
+os.chdir(this_file_path)
+shutil.rmtree(tmp_download_dir_full_path)

Original file line number	Diff line number	Diff line change
`@@ -630,6 +630,9 @@ list_groups (Handle self, PyObject args, PyObject *kwargs) {`
`630`	`630`	`}`
`631`	`631`
`632`	`632`	`const char list_groups_doc[] = PyDoc_STR(`
	`633`	`+ ".. deprecated:: 2.0.2"`
	`634`	+ " Use :func:`list_consumer_groups` and `describe_consumer_groups` instead."
	`635`	`+ "\n"`
`633`	`636`	`".. py:function:: list_groups([group=None], [timeout=-1])\n"`
`634`	`637`	`"\n"`
`635`	`638`	`" Request Group Metadata from cluster.\n"`