Enable CI for the streamstorage python client (#3875)

Author: zymap

.github/workflows/bk-streamstorage-python.yml

@@ -0,0 +1,85 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+name: BookKeeper StreamStorage Python Client
+on:
+  pull_request:
+    branches:
+      - master
+      - branch-*
+    paths:
+      - 'stream/**'
+      - '.github/workflows/bk-streamstorage-python.yml'
+  push:
+    branches:
+      - master
+      - branch-*
+    paths:
+      - 'stream/**'
+      - '.github/workflows/bk-streamstorage-python.yml'
+
+jobs:
+  stream-storage-python-client-unit-tests:
+    name: StreamStorage Python Client Unit Tests
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - name: checkout
+        uses: actions/checkout@v3
+      - name: Tune Runner VM
+        uses: ./.github/actions/tune-runner-vm
+      - name: Test
+        run: ./stream/clients/python/scripts/test.sh
+  
+
+  Stream-storage-python-client-integration-tests:
+    name: StreamStorage Python Client Integration Tests
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - name: checkout
+        uses: actions/checkout@v3
+      - name: Tune Runner VM
+        uses: ./.github/actions/tune-runner-vm
+      - name: Cache local Maven repository
+        id: cache
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/.m2/repository/*/*/*
+            !~/.m2/repository/org/apache/bookkeeper
+            !~/.m2/repository/org/apache/distributedlog
+          key: ${{ runner.os }}-bookkeeper-all-${{ hashFiles('**/pom.xml') }}
+      - name: Set up JDK 11
+        uses: actions/setup-java@v2
+        with:
+          distribution: 'temurin'
+          java-version: 11
+      - name: Set up Maven
+        uses: apache/pulsar-test-infra/setup-maven@master
+        with:
+          maven-version: 3.8.7
+      - name: Build
+        run: mvn -q -T 1C -B -nsu clean install -DskipTests -Dcheckstyle.skip -Dspotbugs.skip -Drat.skip -Dmaven.javadoc.skip
+      - name: Build Test image
+        run: ./stream/clients/python/docker/build-local-image.sh
+      - name: Test
+        run: ./stream/clients/python/scripts/docker_integration_tests.sh
+
+

stream/clients/pom.xml

@@ -29,4 +29,22 @@
   <modules>
     <module>java</module>
   </modules>
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.rat</groupId>
+        <artifactId>apache-rat-plugin</artifactId>
+        <configuration>
+          <excludes>
+            <exclude>**/apache_bookkeeper_client.egg-info/**</exclude>
+            <exclude>**/.nox/**</exclude>
+            <exclude>**/.pytest_cache/**</exclude>
+            <exclude>**/.idea/**</exclude>
+            <exclude>**/build/**</exclude>
+            <exclude>**/proto/**</exclude>
+          </excludes>
+        </configuration>
+      </plugin>
+    </plugins>
+  </build>
 </project>

stream/clients/python/.flake8

@@ -11,6 +11,7 @@
 # limitations under the License.
 
 [flake8]
+max-line-length = 120
 exclude =
   # Exclude generated code.
   **/proto/**

stream/clients/python/.gitignore

@@ -26,3 +26,5 @@ pip-selfcheck.json
 
 # egg-info
 **egg-info/
+
+**/bookkeeper-server-bin.tar.gz

stream/clients/python/README.rst

@@ -1,3 +1,22 @@
+..
+
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+
 Python Client for Apache BookKeeper
 ===================================
 

stream/clients/python/bookkeeper/common/protobuf_helpers.py

@@ -12,10 +12,14 @@
 
 """Helpers for :mod:`protobuf`."""
 
-import collections
 import copy
 import inspect
 
+from collections.abc import Mapping
+from collections.abc import MutableMapping
+from collections.abc import MutableSequence
+from collections import OrderedDict
+
 from google.protobuf import field_mask_pb2
 from google.protobuf import message
 from google.protobuf import wrappers_pb2
@@ -82,7 +86,7 @@ def get_messages(module):
             Message class names as keys, and the Message subclasses themselves
             as values.
     """
-    answer = collections.OrderedDict()
+    answer = OrderedDict()
     for name in dir(module):
         candidate = getattr(module, name)
         if (inspect.isclass(candidate) and
@@ -143,7 +147,7 @@ def get(msg_or_dict, key, default=_SENTINEL):
     # If we get something else, complain.
     if isinstance(msg_or_dict, message.Message):
         answer = getattr(msg_or_dict, key, default)
-    elif isinstance(msg_or_dict, collections.Mapping):
+    elif isinstance(msg_or_dict, Mapping):
         answer = msg_or_dict.get(key, default)
     else:
         raise TypeError(
@@ -166,21 +170,21 @@ def _set_field_on_message(msg, key, value):
     """Set helper for protobuf Messages."""
     # Attempt to set the value on the types of objects we know how to deal
     # with.
-    if isinstance(value, (collections.MutableSequence, tuple)):
+    if isinstance(value, (MutableSequence, tuple)):
         # Clear the existing repeated protobuf message of any elements
         # currently inside it.
         while getattr(msg, key):
             getattr(msg, key).pop()
 
         # Write our new elements to the repeated field.
         for item in value:
-            if isinstance(item, collections.Mapping):
+            if isinstance(item, Mapping):
                 getattr(msg, key).add(**item)
             else:
                 # protobuf's RepeatedCompositeContainer doesn't support
                 # append.
                 getattr(msg, key).extend([item])
-    elif isinstance(value, collections.Mapping):
+    elif isinstance(value, Mapping):
         # Assign the dictionary values to the protobuf message.
         for item_key, item_value in value.items():
             set(getattr(msg, key), item_key, item_value)
@@ -202,7 +206,7 @@ def set(msg_or_dict, key, value):
     """
     # Sanity check: Is our target object valid?
     if (not isinstance(msg_or_dict,
-                       (collections.MutableMapping, message.Message))):
+                       (MutableMapping, message.Message))):
         raise TypeError(
             'set() expected a dict or protobuf message, got {!r}.'.format(
                 type(msg_or_dict)))
@@ -213,12 +217,12 @@ def set(msg_or_dict, key, value):
     # If a subkey exists, then get that object and call this method
     # recursively against it using the subkey.
     if subkey is not None:
-        if isinstance(msg_or_dict, collections.MutableMapping):
+        if isinstance(msg_or_dict, MutableMapping):
             msg_or_dict.setdefault(basekey, {})
         set(get(msg_or_dict, basekey), subkey, value)
         return
 
-    if isinstance(msg_or_dict, collections.MutableMapping):
+    if isinstance(msg_or_dict, MutableMapping):
         msg_or_dict[key] = value
     else:
         _set_field_on_message(msg_or_dict, key, value)