Release 0.4.31 (#457)

* Colab logging (#455)

* Trying setting log level to minumum

* test

* fix

* Update aperturedb/__init__.py

Co-authored-by: Drew Ogle <[email protected]>

* fmt

* Fix for Kaggle issue

---------

Co-authored-by: Drew Ogle <[email protected]>

* expand protobuf support to 4.x (#456)

* expand protobuf support to 4.x
* add shim file
* try to make it performant
* make queryMessage wrap ParseFromString

* Version bump: 0.4.30 to 0.4.31

---------

Co-authored-by: bovlb <[email protected]>
Co-authored-by: Drew Ogle <[email protected]>
Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

Author: 4 people

aperturedb/Connector.py

@@ -25,7 +25,7 @@
 # THE SOFTWARE.
 #
 from __future__ import annotations
-from . import queryMessage_pb2
+from . import queryMessage
 import sys
 import os
 import socket
@@ -359,7 +359,7 @@ def _query(self, query, blob_array = [], try_resume=True):
         else:
             query_str = query
 
-        query_msg = queryMessage_pb2.queryMessage()
+        query_msg = queryMessage.queryMessage()
         # query has .json and .blobs
         query_msg.json = query_str
 
@@ -380,8 +380,8 @@ def _query(self, query, blob_array = [], try_resume=True):
                 if self._send_msg(data):
                     response = self._recv_msg()
                     if response is not None:
-                        querRes = queryMessage_pb2.queryMessage()
-                        querRes.ParseFromString(response)
+                        querRes = queryMessage.queryMessage()
+                        queryMessage.ParseFromString(querRes, response)
                         response_blob_array = [b for b in querRes.blobs]
                         self.last_response = json.loads(querRes.json)
                         break

aperturedb/__init__.py

@@ -9,17 +9,23 @@
 
 logger = logging.getLogger(__name__)
 
-__version__ = "0.4.30"
+__version__ = "0.4.31"
 
 # set log level
-logger.setLevel(logging.DEBUG)
-formatter    = logging.Formatter(
+formatter = logging.Formatter(
     "%(asctime)s : %(levelname)s : %(name)s : %(thread)d : %(lineno)d : %(message)s")
 
 log_file_level = logging.getLevelName(os.getenv("LOG_FILE_LEVEL", "WARN"))
 log_console_level = logging.getLevelName(
     os.getenv("LOG_CONSOLE_LEVEL", "ERROR"))
 
+# Set the logger filter to the minimum (more chatty) of the two handler levels
+# This reduces problems if the environment adds a root handler (e.g. Google Colab)
+logger_level = min(log_file_level, log_console_level)
+if any(log_control in os.environ
+       for log_control in ["LOG_CONSOLE_LEVEL", "LOG_FILE_LEVEL"]):
+    logger.setLevel(logger_level)
+
 # define file handler and set formatter
 error_file_name = "error.${now}.log"
 

aperturedb/queryMessage.py

@@ -0,0 +1,23 @@
+# queryMessage.py - wraps protobuf versions
+import google.protobuf
+
+if google.protobuf.__version__.split(".")[0] == "3":
+    from . import queryMessage3_pb2
+
+    def queryMessage():
+        return queryMessage3_pb2.queryMessage()
+
+    def ParseFromString(msg, data):
+        return msg.ParseFromString(data)
+elif google.protobuf.__version__.split(".")[0] == "4":
+    from . import queryMessage4_pb2
+
+    def queryMessage():
+        return queryMessage4_pb2.queryMessage()
+
+    def ParseFromString(msg, data):
+        # because of https://github.com/protocolbuffers/protobuf/issues/10774
+        return msg.ParseFromString(memoryview(data).tobytes())
+else:
+    raise Exception(
+        f"aperturedb not compatible with {google.protobuf.__version__}")

aperturedb/queryMessage_pb2.py renamed to aperturedb/queryMessage3_pb2.py

@@ -0,0 +1,14 @@
+ApertureDB python library supports version 3 and 4 of protobuf.
+
+These are incompatable implementations in that the proto file that protoc
+compiles into python code will not work between versions.
+
+To solve this we have added a thin wrapper which selects the backend
+implementation based on the system installed version.
+
+This is done because customers use packages alongside aperturedb which require
+python protobuf packages from both the 3.x line and the 4.x line.
+
+To regenerate the files, simply take the queryMessage.proto file, and make
+copies which append the version at the end, then use the matching protoc to
+compile them. Finally place them in this repo.

aperturedb/queryMessage4_pb2.py

@@ -16,11 +16,12 @@ authors = [
 dependencies = [
     # Pin to the bridge version.
     # https://github.com/tensorflow/tensorflow/issues/60320
-    'protobuf==3.20.3',
+    'protobuf >=3.20.3,<5.0.0',
     #Folllowing is needed parallel loaders, and basic things for
     # making the notebooks.
     'scikit-image', 'image', 'requests', 'boto3',
-    'numpy<2', 'matplotlib', 'pandas', 'kaggle', 'google-cloud-storage',
+    # https://github.com/Kaggle/kaggle-api/issues/611
+    'numpy<2', 'matplotlib', 'pandas', 'kaggle!=1.6.15', 'google-cloud-storage',
     'ipython', 'dask[complete]', 'ipywidgets', 'pydantic', 'devtools', 'typer[all]',
     "opencv-python-headless",
     # Pinning this to be able to install google-cloud-bigquery