Merge pull request #5931 from fstagni/80_fixes22

[8.0] from BLOBs to TEXTs

Author: fstagni

src/DIRAC/FrameworkSystem/DB/ComponentMonitoringDB.py

@@ -1,6 +1,5 @@
 """ ComponentMonitoring class is a front-end to the Component monitoring Database
 """
-import six
 from urllib import parse
 
 from DIRAC import gConfig, S_OK, S_ERROR
@@ -84,7 +83,7 @@ def __initializeDB(self):
                     "Version": "VARCHAR(255)",
                     "DIRACVersion": "VARCHAR(255) NOT NULL",
                     "Platform": "VARCHAR(255) NOT NULL",
-                    "Description": "BLOB",
+                    "Description": "TEXT",
                 },
                 "Indexes": {"Component": ["CompId"]},
             }
@@ -95,7 +94,7 @@ def __datetime2str(self, dt):
         """
         This method converts the datetime type to a string type.
         """
-        if isinstance(dt, six.string_types):
+        if isinstance(dt, str):
             return dt
         return "%s-%s-%s %s:%s:%s" % (dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second)
 
@@ -249,9 +248,9 @@ def __getComponents(self, condDict):
         sqlWhere = []
         for field in condDict:
             val = condDict[field]
-            if isinstance(val, six.string_types):
+            if isinstance(val, str):
                 sqlWhere.append("%s='%s'" % (field, val))
-            elif isinstance(val, six.integer_types + (float,)):
+            elif isinstance(val, (int, float)):
                 sqlWhere.append("%s='%s'" % (field, val))
             else:
                 sqlWhere.append("( %s )" % " OR ".join(["%s='%s'" % (field, v) for v in val]))

src/DIRAC/FrameworkSystem/DB/InstalledComponentsDB.py

@@ -1,7 +1,6 @@
 """
 Classes and functions for easier management of the InstalledComponents database
 """
-import six
 import re
 import datetime
 from sqlalchemy import MetaData, Column, Integer, String, DateTime, create_engine, text
@@ -416,7 +415,7 @@ def __filterFields(self, session, table, matchFields=None):
                         toAppend = element
                         if isinstance(toAppend, datetime.datetime):
                             toAppend = toAppend.strftime("%Y-%m-%d %H:%M:%S")
-                        if isinstance(toAppend, six.string_types):
+                        if isinstance(toAppend, str):
                             toAppend = "'%s'" % (toAppend)
                         if i == 0:
                             sql = "%s%s" % (sql, toAppend)
@@ -425,7 +424,7 @@ def __filterFields(self, session, table, matchFields=None):
                     sql = "%s )" % (sql)
                 else:
                     continue
-            elif isinstance(matchFields[key], six.string_types):
+            elif isinstance(matchFields[key], str):
                 sql = "`%s` %s '%s'" % (actualKey, comparison, matchFields[key])
             elif isinstance(matchFields[key], datetime.datetime):
                 sql = "%s %s '%s'" % (actualKey, comparison, matchFields[key].strftime("%Y-%m-%d %H:%M:%S"))

src/DIRAC/FrameworkSystem/DB/NotificationDB.py

@@ -65,7 +65,7 @@ def __initializeDB(self):
                     "Subject": "VARCHAR(255) NOT NULL",
                     "Status": "VARCHAR(64) NOT NULL",
                     "Priority": "VARCHAR(32) NOT NULL",
-                    "Body": "BLOB",
+                    "Body": "TEXT",
                     "Assignee": "VARCHAR(64) NOT NULL",
                     "Notifications": "VARCHAR(128) NOT NULL",
                 },
@@ -87,7 +87,7 @@ def __initializeDB(self):
                     "AlarmId": "INTEGER UNSIGNED NOT NULL",
                     "Timestamp": "DATETIME NOT NULL",
                     "Author": "VARCHAR(64) NOT NULL",
-                    "Comment": "BLOB",
+                    "Comment": "TEXT",
                     "Modifications": "VARCHAR(255)",
                 },
                 "Indexes": {"AlarmID": ["AlarmId"]},
@@ -110,7 +110,7 @@ def __initializeDB(self):
                 "Fields": {
                     "Id": "INTEGER UNSIGNED AUTO_INCREMENT NOT NULL",
                     "User": "VARCHAR(64) NOT NULL",
-                    "Message": "BLOB NOT NULL",
+                    "Message": "TEXT NOT NULL",
                     "Seen": "TINYINT(1) NOT NULL DEFAULT 0",
                     "Expiration": "DATETIME",
                     "Timestamp": "DATETIME",

src/DIRAC/FrameworkSystem/DB/UserProfileDB.py

@@ -1,8 +1,6 @@
 """ UserProfileDB class is a front-end to the User Profile Database
 """
 
-import six
-
 import cachetools
 
 from DIRAC import S_OK, S_ERROR
@@ -51,7 +49,7 @@ class UserProfileDB(DB):
                 "VOId": "INTEGER",
                 "Profile": "VARCHAR(255) NOT NULL",
                 "VarName": "VARCHAR(255) NOT NULL",
-                "Data": "BLOB",
+                "Data": "TEXT",
                 "ReadAccess": 'VARCHAR(10) DEFAULT "USER"',
                 "PublishAccess": 'VARCHAR(10) DEFAULT "USER"',
             },
@@ -260,7 +258,8 @@ def retrieveVarById(self, userIds, ownerIds, profileName, varName):
             return result
         data = result["Value"]
         if len(data) > 0:
-            return S_OK(data[0][0].decode())
+            # TODO: The decode is only needed in DIRAC v8.0.x while moving from BLOB -> TEXT
+            return S_OK(data[0][0].decode() if isinstance(data[0][0], bytes) else data[0][0])
         return S_ERROR("No data for userIds %s profileName %s varName %s" % (userIds, profileName, varName))
 
     def retrieveAllUserVarsById(self, userIds, profileName):
@@ -278,7 +277,12 @@ def retrieveAllUserVarsById(self, userIds, profileName):
         if not result["OK"]:
             return result
         data = result["Value"]
-        return S_OK({k: v.decode() for k, v in data})
+        try:
+            # TODO: This is only needed in DIRAC v8.0.x while moving from BLOB -> TEXT
+            allUserDataDict = {k: v.decode() for k, v in data}
+        except AttributeError:
+            allUserDataDict = {k: v for k, v in data}
+        return S_OK(allUserDataDict)
 
     def retrieveUserProfilesById(self, userIds):
         """
@@ -289,12 +293,15 @@ def retrieveUserProfilesById(self, userIds):
         result = self._query(selectSQL)
         if not result["OK"]:
             return result
-        data = result["Value"]
         dataDict = {}
-        for profile, varName, data in data:
+        for profile, varName, data in result["Value"]:
             if profile not in dataDict:
                 dataDict[profile] = {}
-            dataDict[profile][varName] = data.decode()
+            try:
+                # TODO: This is only needed in DIRAC v8.0.x while moving from BLOB -> TEXT
+                dataDict[profile][varName] = data.decode()
+            except AttributeError:
+                dataDict[profile][varName] = data
         return S_OK(dataDict)
 
     def retrieveVarPermsById(self, userIds, ownerIds, profileName, varName):
@@ -489,7 +496,7 @@ def deleteVar(self, userName, userGroup, profileName, varName):
         return self.deleteVarByUserId(userIds, profileName, varName)
 
     def __profilesCondGenerator(self, value, varType, initialValue=False):
-        if isinstance(value, six.string_types):
+        if isinstance(value, str):
             value = [value]
         ids = []
         if initialValue:

src/DIRAC/ProductionSystem/DB/ProductionDB.py

@@ -4,7 +4,6 @@
     in order to automate the task of transformation preparation for high level productions.
 """
 # # imports
-import six
 import json
 import threading
 
@@ -146,10 +145,10 @@ def getProductions(
         webList = []
         resultList = []
         for row in res["Value"]:
-            # HACK: Description should be converted to a text type
+            # TODO: remove, as Description should have been converted to a text type
             row = [item.decode() if isinstance(item, bytes) else item for item in row]
             # Prepare the structure for the web
-            rList = [str(item) if not isinstance(item, six.integer_types) else item for item in row]
+            rList = [str(item) if not isinstance(item, int) else item for item in row]
             prodDict = dict(zip(self.PRODPARAMS, row))
             webList.append(rList)
             resultList.append(prodDict)
@@ -181,7 +180,7 @@ def getProductionParameters(self, prodName, parameters, connection=False):
         :param str prodName: the Production name or ID
         :param str parameters: any valid production parameter in self.PRODPARAMS
         """
-        if isinstance(parameters, six.string_types):
+        if isinstance(parameters, str):
             parameters = [parameters]
         res = self.getProduction(prodName, connection=connection)
         if not res["OK"]:
@@ -215,7 +214,7 @@ def getProductionStep(self, stepID, connection=False):
         if not res["Value"]:
             return S_ERROR("ProductionStep %s did not exist" % str(stepID))
         row = res["Value"][0]
-        # HACK: LongDescription and Body should be converted to a text type
+        # TODO: remove, as Description and body should have been converted to a text type
         row = [item.decode() if isinstance(item, bytes) else item for item in row]
         return S_OK(row)
 
@@ -319,7 +318,7 @@ def getProductionTransformations(
         resultList = []
         for row in res["Value"]:
             # Prepare the structure for the web
-            rList = [str(item) if not isinstance(item, six.integer_types) else item for item in row]
+            rList = [str(item) if not isinstance(item, int) else item for item in row]
             transDict = dict(zip(self.TRANSPARAMS, row))
             webList.append(rList)
             resultList.append(transDict)
@@ -619,7 +618,7 @@ def _getProductionID(self, prodName, connection=False):
             prodName = int(prodName)
             cmd = "SELECT ProductionID from Productions WHERE ProductionID=%d;" % prodName
         except Exception:
-            if not isinstance(prodName, six.string_types):
+            if not isinstance(prodName, str):
                 return S_ERROR("Production should be ID or name")
             cmd = "SELECT ProductionID from Productions WHERE ProductionName='%s';" % prodName
         res = self._query(cmd, connection)

src/DIRAC/ProductionSystem/DB/ProductionDB.sql

@@ -22,7 +22,7 @@ DROP TABLE IF EXISTS Productions;
 CREATE TABLE Productions(
     ProductionID INTEGER NOT NULL AUTO_INCREMENT,
     ProductionName VARCHAR(255) NOT NULL,
-    Description LONGBLOB,
+    Description LONGTEXT,
     CreationDate DATETIME,
     LastUpdate DATETIME,
     AuthorDN VARCHAR(255) NOT NULL,
@@ -38,8 +38,8 @@ CREATE TABLE ProductionSteps(
     StepID INTEGER NOT NULL AUTO_INCREMENT,
     Name VARCHAR(255),
     Description VARCHAR(255),
-    LongDescription BLOB,
-    Body LONGBLOB,
+    LongDescription TEXT,
+    Body LONGTEXT,
     Type CHAR(32) DEFAULT 'Simulation',
     Plugin CHAR(32) DEFAULT 'None',
     AgentType CHAR(32) DEFAULT 'Manual',