feat: add bytes decoding, tests

Author: TaykYoku

src/DIRAC/ConfigurationSystem/Client/ConfigurationClient.py

@@ -29,7 +29,7 @@ def getCompressedData(self):
         :rtype: str
         """
         retVal = self.executeRPC("getCompressedData")
-        if retVal["OK"]:
+        if retVal["OK"] and isinstance(retVal["Value"], six.string_types):
             retVal["Value"] = b64decode(retVal["Value"])
         return retVal
 
@@ -41,7 +41,7 @@ def getCompressedDataIfNewer(self, sClientVersion):
         :returns: Configuration data, if changed, compressed
         """
         retVal = self.executeRPC("getCompressedDataIfNewer", sClientVersion)
-        if retVal["OK"] and "data" in retVal["Value"]:
+        if retVal["OK"] and isinstance(retVal["Value"].get("data"), six.string_types):
             retVal["Value"]["data"] = b64decode(retVal["Value"]["data"])
         return retVal
 

src/DIRAC/ConfigurationSystem/Service/TornadoConfigurationHandler.py

@@ -12,6 +12,7 @@
 
 __RCSID__ = "$Id$"
 
+import six
 from base64 import b64encode, b64decode
 
 from DIRAC import S_OK, S_ERROR, gLogger
@@ -79,7 +80,9 @@ def export_commitNewData(self, sData):
         credDict = self.getRemoteCredentials()
         if "DN" not in credDict or "username" not in credDict:
             return S_ERROR("You must be authenticated!")
-        sData = b64decode(sData)
+        # TODO: in 8.0 remove it
+        if isinstance(sData, six.string_types):
+            sData = b64decode(sData)
         return self.ServiceInterface.updateConfiguration(sData, credDict["username"])
 
     def export_writeEnabled(self):

src/DIRAC/Core/Utilities/JEncode.py

@@ -4,6 +4,8 @@
 from __future__ import division
 from __future__ import print_function
 
+import six
+from base64 import b64encode, b64decode
 import datetime
 import json
 
@@ -88,6 +90,21 @@ class DJSONEncoder(json.JSONEncoder):
     tuple, datetime, and any object inheriting from JSerializable
     """
 
+    # TODO: remove it in 8.0
+    def encode(self, obj):
+        """This method for backward compatability with python 2 code"""
+        if six.PY2:
+            # Json can't distinguish bytes or str in python 2, so it's trying to encode with utf-8/ascii
+            try:
+                # Let's leave this conversion for backward compatibility, although
+                # it does not guarantee the return of the same data when decoding
+                return super(DJSONEncoder, self).encode(obj)
+            except UnicodeDecodeError as e:
+                # If the encoding type does not match we have an exception
+                # that json does not try to fix using default method
+                obj = self.default(obj)  # Let's use base64 in this case
+        return super(DJSONEncoder, self).encode(obj)
+
     def default(self, obj):  # pylint: disable=method-hidden
         """Add supports for datetime and JSerializable class to default json
 
@@ -105,7 +122,9 @@ def default(self, obj):  # pylint: disable=method-hidden
         # if the object inherits from JSJerializable, try to serialize it
         elif isinstance(obj, JSerializable):
             return obj._toJSON()  # pylint: disable=protected-access
-
+        # if the object a bytes, decode it
+        elif isinstance(obj, bytes):
+            return {"__dCls": "b64", "obj": b64encode(obj).decode()}
         # otherwise, let the parent do
         return super(DJSONEncoder, self).default(obj)
 
@@ -140,6 +159,8 @@ def dict_to_object(dataDict):
             return datetime.datetime.strptime(dataDict["obj"], DATETIME_DEFAULT_FORMAT)
         elif className == "date":
             return datetime.datetime.strptime(dataDict["obj"], DATETIME_DEFAULT_DATE_FORMAT).date()
+        elif className == "b64":
+            return b64decode(dataDict["obj"])
         elif className:
             import importlib
 

src/DIRAC/Core/Utilities/test/Test_Encode.py

@@ -21,6 +21,7 @@
 
 from hypothesis import given, settings, HealthCheck
 from hypothesis.strategies import (
+    binary,
     builds,
     integers,
     lists,
@@ -59,7 +60,7 @@
     "jsonTuple",
     "mixTuple",
     "mixTuple (DIRAC_USE_JSON_DECODE=Yes)",
-    "mixTuple (DIRAC_USE_JSON_ENCODE=Yes",
+    "mixTuple (DIRAC_USE_JSON_ENCODE=Yes)",
 )
 
 enc_dec_imp_without_json = (disetTuple, (mixTuple, "No", "No"), (mixTuple, "Yes", "No"))
@@ -118,6 +119,8 @@ def test_everyBaseTypeIsTested():
     for encodeFunc in g_dEncodeFunctions.values():
         testFuncName = ("test_BaseType_%s" % encodeFunc.__name__).replace("encode", "")
         globals()[testFuncName]
+    # We forgot about binary types
+    globals()["test_BaseType_Bytes"]
 
 
 def agnosticTestFunction(enc_dec_tuple, data):
@@ -167,6 +170,14 @@ def enc_dec_without_json(request, monkeypatch):
     return base_enc_dec(request, monkeypatch)
 
 
+@settings(suppress_health_check=function_scoped)
+@given(data=binary())
+def test_BaseType_Bytes(data):
+    """Test for bytes. Test JEncode with python 3 ONLY, since DEncode distorts the result"""
+    if six.PY3:
+        agnosticTestFunction(jsonTuple, data)
+
+
 @settings(suppress_health_check=function_scoped)
 @given(data=booleans())
 def test_BaseType_Bool(enc_dec, data):