Skip to content

Commit d580658

Browse files
chrisburrchrisburr
authored
Merge pull request #5783 from fstagni/80_SB_DB
[80] Minor fixes for SandboxStore, and integration tests
2 parents ca12a95 + 212bad0 commit d580658

16 files changed

+50
-76
lines changed

‎src/DIRAC/WorkloadManagementSystem/ConfigTemplate.cfg‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,7 @@ Services
101101
Enable = No
102102
PilotsLoggingQueue = serviceURL::QueueType::QueueName
103103
}
104+
##BEGIN SandboxStore
104105
SandboxStore
105106
{
106107
Port = 9196
@@ -121,6 +122,7 @@ Services
121122
}
122123
}
123124
}
125+
##END
124126
OptimizationMind
125127
{
126128
Port = 9175

‎src/DIRAC/WorkloadManagementSystem/DB/SandboxMetadataDB.py‎

Lines changed: 12 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ def __init__(self):
1212
DB.__init__(self, "SandboxMetadataDB", "WorkloadManagement/SandboxMetadataDB")
1313
result = self.__initializeDB()
1414
if not result["OK"]:
15-
raise RuntimeError("Can't create tables: %s" % result["Message"])
15+
raise RuntimeError(f"Can't create tables: {result['Message']}")
1616
self.__assignedSBGraceDays = 0
1717
self.__unassignedSBGraceDays = 15
1818

@@ -73,7 +73,7 @@ def __initializeDB(self):
7373

7474
return self._createTables(tablesToCreate)
7575

76-
def registerAndGetOwnerId(self, owner, ownerDN, ownerGroup):
76+
def __registerAndGetOwnerId(self, owner, ownerDN, ownerGroup):
7777
"""
7878
Get the owner ID and register it if it's not there
7979
"""
@@ -112,7 +112,7 @@ def registerAndGetSandbox(self, owner, ownerDN, ownerGroup, sbSE, sbPFN, size=0)
112112
Register a new sandbox in the metadata catalog
113113
Returns ( sbid, newSandbox )
114114
"""
115-
result = self.registerAndGetOwnerId(owner, ownerDN, ownerGroup)
115+
result = self.__registerAndGetOwnerId(owner, ownerDN, ownerGroup)
116116
if not result["OK"]:
117117
return result
118118
ownerId = result["Value"]
@@ -135,7 +135,7 @@ def registerAndGetSandbox(self, owner, ownerDN, ownerGroup, sbSE, sbPFN, size=0)
135135
if not result["OK"]:
136136
return result
137137
if not result["Value"]:
138-
return S_ERROR("Location %s already exists but doesn't belong to the user or setup")
138+
return S_ERROR("SandBox already exists but doesn't belong to the user or setup")
139139
sbId = result["Value"][0][0]
140140
self.accessedSandboxById(sbId)
141141
return S_OK((sbId, False))
@@ -144,7 +144,7 @@ def registerAndGetSandbox(self, owner, ownerDN, ownerGroup, sbSE, sbPFN, size=0)
144144
return S_OK((result["lastRowId"], True))
145145
result = self._query("SELECT LAST_INSERT_ID()")
146146
if not result["OK"]:
147-
return S_ERROR("Can't determine sand box id after insertion")
147+
return S_ERROR("Can't determine sandbox id after insertion")
148148
return S_OK((result["Value"][0][0], True))
149149

150150
def accessedSandboxById(self, sbId):
@@ -174,12 +174,12 @@ def assignSandboxesToEntities(self, enDict, requesterName, requesterGroup, enSet
174174
for entityId in enDict:
175175
for sbTuple in enDict[entityId]:
176176
if not isinstance(sbTuple, (tuple, list)):
177-
return S_ERROR("Entry for entity %s is not a itterable of tuples/lists" % entityId)
177+
return S_ERROR(f"Entry for entity {entityId} is not an iterable of tuples/lists")
178178
if len(sbTuple) != 2:
179-
return S_ERROR("SB definition is not ( SBLocation, Type )! It's '%s'" % str(sbTuple))
179+
return S_ERROR(f"SB definition is not ( SBLocation, Type )! It's {sbTuple}")
180180
SBLocation = sbTuple[0]
181181
if SBLocation.find("SB:") != 0:
182-
return S_ERROR("%s doesn't seem to be a sandbox" % SBLocation)
182+
return S_ERROR(f"{SBLocation} doesn't seem to be a sandbox")
183183
SBLocation = SBLocation[3:]
184184
splitted = List.fromChar(SBLocation, "|")
185185
if len(splitted) < 2:
@@ -197,7 +197,8 @@ def assignSandboxesToEntities(self, enDict, requesterName, requesterGroup, enSet
197197
insertValues = []
198198
if not result["OK"]:
199199
self.log.warn(
200-
"Cannot find id for %s:%s with requester %s@%s" % (SEName, SEPFN, requesterName, requesterGroup)
200+
f"Cannot find id for {SEName}:",
201+
f"{SEPFN} with requester {requesterName}@{requesterGroup}: {result['Message']}",
201202
)
202203
else:
203204
sbId = result["Value"]
@@ -273,7 +274,7 @@ def unassignEntities(self, entitiesDict, requesterName, requesterGroup):
273274
escapedSetup = self._escapeString(entitySetup)["Value"]
274275
result = self.__filterEntitiesByRequester(entitiesIds, escapedSetup, requesterName, requesterGroup)
275276
if not result["OK"]:
276-
gLogger.error("Cannot filter entities: %s" % result["Message"])
277+
gLogger.error("Cannot filter entities", result["Message"])
277278
continue
278279
ids = result["Value"]
279280
if not ids:
@@ -283,7 +284,7 @@ def unassignEntities(self, entitiesDict, requesterName, requesterGroup):
283284
sqlCmd = "DELETE FROM `sb_EntityMapping` WHERE %s" % " AND ".join(sqlCond)
284285
result = self._update(sqlCmd)
285286
if not result["OK"]:
286-
gLogger.error("Cannot unassign entities: %s" % result["Message"])
287+
gLogger.error("Cannot unassign entities", result["Message"])
287288
else:
288289
updated += 1
289290
return S_OK(updated)

‎src/DIRAC/WorkloadManagementSystem/Service/SandboxStoreHandler.py‎

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,11 @@
11
""" SandboxHandler is the implementation of the Sandbox service
22
in the DISET framework
3+
4+
.. literalinclude:: ../ConfigTemplate.cfg
5+
:start-after: ##BEGIN SandboxStore
6+
:end-before: ##END
7+
:dedent: 2
8+
:caption: SandboxStore options
39
"""
410
import os
511
import time
@@ -36,7 +42,7 @@ def initializeHandler(cls, serviceInfoDict):
3642
cls.sandboxDB = result["Value"]()
3743

3844
except RuntimeError as excp:
39-
return S_ERROR("Can't connect to DB: %s" % excp)
45+
return S_ERROR("Can't connect to DB: %s" % repr(excp))
4046
return S_OK()
4147

4248
def initialize(self):
@@ -95,7 +101,7 @@ def transfer_fromClient(self, fileId, token, fileSize, fileHelper):
95101
else:
96102
extension = ""
97103
aHash = fileId
98-
gLogger.info("Upload requested for %s [%s]" % (aHash, extension))
104+
gLogger.info("Upload requested", f"for {aHash} [{extension}]")
99105

100106
credDict = self.getRemoteCredentials()
101107
sbPath = self.__getSandboxPath("%s.%s" % (aHash, extension))
@@ -123,10 +129,10 @@ def transfer_fromClient(self, fileId, token, fileSize, fileHelper):
123129
# Write to local file
124130
result = self.__networkToFile(fileHelper, hdPath)
125131
if not result["OK"]:
126-
gLogger.error("Error while receiving sandbox file", "%s" % result["Message"])
132+
gLogger.error("Error while receiving sandbox file", result["Message"])
127133
return result
128134
hdPath = result["Value"]
129-
gLogger.info("Wrote sandbox to file %s" % hdPath)
135+
gLogger.info("Wrote sandbox to file", hdPath)
130136
# Check hash!
131137
if fileHelper.getHash() != aHash:
132138
self.__secureUnlinkFile(hdPath)
@@ -141,7 +147,7 @@ def transfer_fromClient(self, fileId, token, fileSize, fileHelper):
141147
return result
142148
sbPath = result["Value"][1]
143149
# Register!
144-
gLogger.info("Registering sandbox in the DB with", "SB:%s|%s" % (self.__seNameToUse, sbPath))
150+
gLogger.info("Registering sandbox in the DB with", f"SB:{self.__seNameToUse}|{sbPath}")
145151
result = self.sandboxDB.registerAndGetSandbox(
146152
credDict["username"],
147153
credDict["DN"],
@@ -154,7 +160,7 @@ def transfer_fromClient(self, fileId, token, fileSize, fileHelper):
154160
self.__secureUnlinkFile(hdPath)
155161
return result
156162

157-
sbURL = "SB:%s|%s" % (self.__seNameToUse, sbPath)
163+
sbURL = f"SB:{self.__seNameToUse}|{sbPath}"
158164
assignTo = dict([(key, [(sbURL, assignTo[key])]) for key in assignTo])
159165
result = self.export_assignSandboxesToEntities(assignTo)
160166
if not result["OK"]:
@@ -521,7 +527,7 @@ def __deleteSandboxFromExternalBackend(self, SEName, SEPFN):
521527
return ReqClient().putRequest(request)
522528
except Exception as e:
523529
gLogger.exception("Exception while setting deletion request")
524-
return S_ERROR("Cannot set deletion request: %s" % str(e))
530+
return S_ERROR(f"Cannot set deletion request: {e}")
525531
else:
526532
gLogger.info("Deleting external Sandbox")
527533
try:

‎tests/Integration/WorkloadManagementSystem/Test_Client_WMS.py‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -25,10 +25,6 @@
2525

2626
# pylint: disable=protected-access,wrong-import-position,invalid-name
2727

28-
from __future__ import print_function
29-
from __future__ import absolute_import
30-
from __future__ import division
31-
3228
import unittest
3329
import sys
3430
import datetime

‎tests/Integration/WorkloadManagementSystem/Test_JobDB.py‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -6,16 +6,10 @@
66

77
# pylint: disable=wrong-import-position
88

9-
from __future__ import print_function
10-
from __future__ import absolute_import
11-
from __future__ import division
12-
139
from datetime import datetime, timedelta
1410

1511
import pytest
1612

17-
__RCSID__ = "$Id$"
18-
1913
from DIRAC.Core.Base.Script import parseCommandLine
2014

2115
parseCommandLine()

‎tests/Integration/WorkloadManagementSystem/Test_JobLoggingDB.py‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,5 @@
11
""" This test only need the JobLoggingDB to be present
22
"""
3-
from __future__ import absolute_import
4-
from __future__ import division
5-
from __future__ import print_function
6-
73
# pylint: disable=invalid-name,wrong-import-position
84

95
import unittest

‎tests/Integration/WorkloadManagementSystem/Test_JobParameters_MySQLandES.py‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,14 +4,9 @@
44
flag in /Operations/[]/Services/JobMonitoring/useESForJobParametersFlag
55
"""
66

7-
from __future__ import print_function, absolute_import
8-
from __future__ import division
9-
107
import os
118
import time
129

13-
__RCSID__ = "$Id$"
14-
1510
from DIRAC.Core.Utilities.DIRACScript import DIRACScript as Script
1611

1712
Script.parseCommandLine()

‎tests/Integration/WorkloadManagementSystem/Test_JobWrapper.py‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,5 @@
11
""" JobWrapper test
22
"""
3-
from __future__ import absolute_import
4-
from __future__ import division
5-
from __future__ import print_function
6-
73
import unittest
84
import os
95
import sys

‎tests/Integration/WorkloadManagementSystem/Test_PilotAgentsDB.py‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,6 @@
33
Suggestion: for local testing, run this with::
44
python -m pytest -c ../pytest.ini -vv tests/Integration/WorkloadManagementSystem/Test_PilotAgentsDB.py
55
"""
6-
from __future__ import absolute_import
7-
from __future__ import division
8-
from __future__ import print_function
9-
106
# pylint: disable=wrong-import-position
117

128
from DIRAC.Core.Base.Script import parseCommandLine

‎tests/Integration/WorkloadManagementSystem/Test_PilotsClient.py‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,10 +9,6 @@
99
this is pytest!
1010
1111
"""
12-
from __future__ import absolute_import
13-
from __future__ import division
14-
from __future__ import print_function
15-
1612
from DIRAC.Core.Base.Script import parseCommandLine
1713

1814
parseCommandLine()

0 commit comments

Comments
 (0)