docs: reinstate class members and Avro, improve docs, fix doc warnings.

Author: edenhill

confluent_kafka/admin/__init__.py

@@ -184,8 +184,8 @@ def set_config(self, name, value, overwrite=True):
 
 class AdminClient (_AdminClientImpl):
     """
-    The Kafka AdminClient provides admin operations for Kafka brokers,
-    topics, groups, and other resource types supported by the broker.
+    AdminClient provides admin operations for Kafka brokers, topics, groups,
+    and other resource types supported by the broker.
 
     The Admin API methods are asynchronous and returns a dict of
     concurrent.futures.Future objects keyed by the entity.
@@ -519,10 +519,13 @@ class TopicMetadata (object):
 
     This class is typically not user instantiated.
 
-    :ivar str topic: Topic name.
+    :ivar str -topic: Topic name.
     :ivar dict partitions: Map of partitions indexed by partition id. Value is PartitionMetadata object.
-    :ivar KafkaError error: Topic error, or None. Value is a KafkaError object.
+    :ivar KafkaError -error: Topic error, or None. Value is a KafkaError object.
     """
+    # The dash in "-topic" and "-error" is needed to circumvent a
+    # Sphinx issue where it tries to reference the same instance variable
+    # on other classes which raises a warning/error.
     def __init__(self):
         self.topic = None
         self.partitions = {}
@@ -548,7 +551,7 @@ class PartitionMetadata (object):
     :ivar int leader: Current leader broker for this partition, or -1.
     :ivar list(int) replicas: List of replica broker ids for this partition.
     :ivar list(int) isrs: List of in-sync-replica broker ids for this partition.
-    :ivar KafkaError error: Partition error, or None. Value is a KafkaError object.
+    :ivar KafkaError -error: Partition error, or None. Value is a KafkaError object.
 
     :warning: Depending on cluster state the broker ids referenced in
               leader, replicas and isrs may temporarily not be reported

confluent_kafka/src/Admin.c

@@ -1540,7 +1540,7 @@ PyTypeObject AdminType = {
         "\n"
         ".. py:function:: Admin(**kwargs)\n"
         "\n"
-        "  Create new AdminClient instance using provided configuration dict.\n"
+        "  Create a new AdminClient instance using the provided configuration dict.\n"
         "\n"
         "This class should not be used directly, use confluent_kafka.AdminClient\n."
         "\n"

confluent_kafka/src/Consumer.c

@@ -1421,27 +1421,16 @@ PyTypeObject ConsumerType = {
 	0,                         /*tp_as_buffer*/
 	Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE |
 	Py_TPFLAGS_HAVE_GC, /*tp_flags*/
-        "High-level Kafka Consumer\n"
+        "A high-level Apache Kafka Consumer\n"
         "\n"
         ".. py:function:: Consumer(config)\n"
         "\n"
-        "  :param dict config: Configuration properties. "
-        "At a minimum ``group.id`` **must** be set,"
-        " ``bootstrap.servers`` **should** be set."
-        "\n"
-        "Create new Consumer instance using provided configuration dict.\n"
-        "\n"
-        " Special configuration properties:\n"
-        "   ``on_commit``: Optional callback will be called when a commit "
-        "request has succeeded or failed.\n"
-        "\n"
-        "\n"
-        ".. py:function:: on_commit(err, partitions)\n"
-        "\n"
-        "  :param KafkaError err: Commit error object, or None on success.\n"
-        "  :param list(TopicPartition) partitions: List of partitions with "
-        "their committed offsets or per-partition errors.\n"
-        "\n"
+        "Create a new Consumer instance using the provided configuration *dict* ("
+        "including properties and callback functions). "
+        "See :ref:`pythonclient_configuration` for more information."
+        "\n\n"
+        ":param dict config: Configuration properties. At a minimum "
+        "``group.id`` **must** be set, ``bootstrap.servers`` **should** be set."
         "\n", /*tp_doc*/
 	(traverseproc)Consumer_traverse, /* tp_traverse */
 	(inquiry)Consumer_clear, /* tp_clear */

confluent_kafka/src/Producer.c

@@ -552,7 +552,7 @@ PyTypeObject ProducerType = {
         "\n"
         "  :param dict config: Configuration properties. At a minimum ``bootstrap.servers`` **should** be set\n"
         "\n"
-        "  Create new Producer instance using provided configuration dict.\n"
+        "  Create a new Producer instance using the provided configuration dict.\n"
         "\n"
         "\n"
         ".. py:function:: len()\n"

confluent_kafka/src/confluent_kafka.c

@@ -486,21 +486,21 @@ static PyMethodDef Message_methods[] = {
 	},
 	{ "timestamp", (PyCFunction)Message_timestamp, METH_NOARGS,
           "Retrieve timestamp type and timestamp from message.\n"
-          "The timestamp type is one of:\n"
-          " * :py:const:`TIMESTAMP_NOT_AVAILABLE`"
-          " - Timestamps not supported by broker\n"
-          " * :py:const:`TIMESTAMP_CREATE_TIME` "
-          " - Message creation time (or source / producer time)\n"
-          " * :py:const:`TIMESTAMP_LOG_APPEND_TIME` "
-          " - Broker receive time\n"
+          "The timestamp type is one of:\n\n"
+          "  * :py:const:`TIMESTAMP_NOT_AVAILABLE` "
+          "- Timestamps not supported by broker.\n"
+          "  * :py:const:`TIMESTAMP_CREATE_TIME` "
+          "- Message creation time (or source / producer time).\n"
+          "  * :py:const:`TIMESTAMP_LOG_APPEND_TIME` "
+          "- Broker receive time.\n"
           "\n"
-          "The returned timestamp should be ignored if the timestamp type is "
+          "  The returned timestamp should be ignored if the timestamp type is "
           ":py:const:`TIMESTAMP_NOT_AVAILABLE`.\n"
           "\n"
           "  The timestamp is the number of milliseconds since the epoch (UTC).\n"
           "\n"
-          "  Timestamps require broker version 0.10.0.0 or later and \n"
-          "  ``{'api.version.request': True}`` configured on the client.\n"
+          "  Timestamps require broker version 0.10.0.0 or later and "
+          "``{'api.version.request': True}`` configured on the client.\n"
           "\n"
 	  "  :returns: tuple of message timestamp type, and timestamp.\n"
 	  "  :rtype: (int, int)\n"

docs/index.rst

@@ -1,79 +1,98 @@
+The confluent_kafka API
+=======================
 
-Welcome to Confluent's Python client for Apache Kafka documentation
-===================================================================
+A reliable, performant and feature rich Python client for Apache Kafka v0.8 and above.
 
-Indices and tables
-==================
+Clients
+   - :ref:`Consumer <pythonclient_consumer>`
+   - :ref:`Producer <pythonclient_producer>`
+   - :ref:`AdminClient <pythonclient_adminclient>`
+
+
+Supporting classes
+    - :ref:`Message <pythonclient_message>`
+    - :ref:`TopicPartition <pythonclient_topicpartition>`
+    - :ref:`KafkaError <pythonclient_kafkaerror>`
+    - :ref:`KafkaException <pythonclient_kafkaexception>`
+    - :ref:`ThrottleEvent <pythonclient_throttleevent>`
+    - :ref:`Avro <pythonclient_avro>`
 
-* :ref:`genindex`
 
-:mod:`confluent_kafka` --- Confluent's Python client for Apache Kafka
-*********************************************************************
+:ref:`genindex`
 
-.. automodule:: confluent_kafka
-   :synopsis: Confluent's Python client for Apache Kafka.
 
+.. _pythonclient_consumer:
 
-********
 Consumer
-********
+========
 
 .. autoclass:: confluent_kafka.Consumer
+   :members:
 
+.. _pythonclient_producer:
 
-********
 Producer
-********
+========
 
 .. autoclass:: confluent_kafka.Producer
+   :members:
 
+.. _pythonclient_adminclient:
 
-*****
-Admin
-*****
+AdminClient
+===========
 
 .. automodule:: confluent_kafka.admin
+   :members:
 
+.. _pythonclient_avro:
 
-.. autoclass:: confluent_kafka.admin.NewTopic
-.. autoclass:: confluent_kafka.admin.NewPartitions
-
-****
 Avro
-****
+====
 
 .. automodule:: confluent_kafka.avro
+   :members:
 
+Supporting Classes
+==================
 
-.. autoclass:: confluent_kafka.avro.CachedSchemaRegistryClient
-
+.. _pythonclient_message:
 
 *******
 Message
 *******
 
 .. autoclass:: confluent_kafka.Message
+   :members:
 
+.. _pythonclient_topicpartition:
 
 **************
 TopicPartition
 **************
 
 .. autoclass:: confluent_kafka.TopicPartition
+   :members:
 
 
+.. _pythonclient_kafkaerror:
+
 **********
 KafkaError
 **********
 
 .. autoclass:: confluent_kafka.KafkaError
+   :members:
+
 
+.. _pythonclient_kafkaexception:
 
 **************
 KafkaException
 **************
 
 .. autoclass:: confluent_kafka.KafkaException
+   :members:
 
 
 ******
@@ -87,16 +106,22 @@ Logical offset constants:
  * :py:const:`OFFSET_STORED` - Use stored/committed offset
  * :py:const:`OFFSET_INVALID` - Invalid/Default offset
 
+
+.. _pythonclient_throttleevent:
+
 *************
 ThrottleEvent
 *************
 
 .. autoclass:: confluent_kafka.ThrottleEvent
+   :members:
 
 
+.. _pythonclient_configuration:
 
 Configuration
 =============
+
 Configuration of producer and consumer instances is performed by
 providing a dict of configuration properties to the instance constructor, e.g.::
 
@@ -117,7 +142,7 @@ The Python bindings also provide some additional configuration properties:
   properties that are applied to all used topics for the instance. **DEPRECATED:**
   topic configuration should now be specified in the global top-level configuration.
 
-* ``error_cb(kafka.KafkaError)``: Callback for generic/global error events. This callback is served upon calling
+* ``error_cb(kafka.KafkaError)``: Callback for generic/global error events, these errors are typically to be considered informational since the client will automatically try to recover. This callback is served upon calling
   ``client.poll()`` or ``producer.flush()``.
 
 * ``throttle_cb(confluent_kafka.ThrottleEvent)``: Callback for throttled request reporting.
@@ -138,8 +163,11 @@ The Python bindings also provide some additional configuration properties:
   callback. The ``msg.headers()`` will return None even if the original message
   had headers set. This callback is served upon calling ``producer.poll()`` or ``producer.flush()``.
 
-* ``on_commit(kafka.KafkaError, list(kafka.TopicPartition))`` (**Consumer**): Callback used to indicate success or failure
-  of asynchronous and automatic commit requests. This callback is served upon calling ``consumer.poll()``. Is not triggered for synchronous commits.
+* ``on_commit(kafka.KafkaError, list(kafka.TopicPartition))`` (**Consumer**): Callback used to indicate
+  success or failure of asynchronous and automatic commit requests. This callback is served upon calling
+  ``consumer.poll()``. Is not triggered for synchronous commits. Callback arguments: *KafkaError* is the
+  commit error, or None on success. *list(TopicPartition)* is the list of partitions with their committed
+  offsets or per-partition errors.
 
 * ``logger=logging.Handler`` kwarg: forward logs from the Kafka client to the
   provided ``logging.Handler`` instance.