Skip to content

Commit 20001c5

Browse files
authored
Merge pull request #6864 from chaen/v8.0_FIX_moreFTS3Perf
[8.0] More FTS3DB perf improvments
2 parents 607214f + 8f27750 commit 20001c5

File tree

4 files changed

+89
-174
lines changed

4 files changed

+89
-174
lines changed

src/DIRAC/DataManagementSystem/Client/FTS3Job.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,9 @@ class FTS3Job(JSerializable):
4545
]
4646

4747
FINAL_STATES = ["Canceled", "Failed", "Finished", "Finisheddirty"]
48+
49+
# This field is only used for optimizing sql queries (`in`` instead of `not in`)
50+
NON_FINAL_STATES = list(set(ALL_STATES) - set(FINAL_STATES))
4851
INIT_STATE = "Submitted"
4952

5053
# END states

src/DIRAC/DataManagementSystem/DB/FTS3DB.py

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -310,7 +310,7 @@ def getActiveJobs(self, limit=20, lastMonitor=None, jobAssignmentTag="Assigned")
310310
ftsJobsQuery = (
311311
session.query(FTS3Job)
312312
.join(FTS3Operation)
313-
.filter(~FTS3Job.status.in_(FTS3Job.FINAL_STATES))
313+
.filter(FTS3Job.status.in_(FTS3Job.NON_FINAL_STATES))
314314
.filter(FTS3Job.assignment.is_(None))
315315
.filter(FTS3Operation.assignment.is_(None))
316316
)
@@ -687,12 +687,11 @@ def deleteFinalOperations(self, limit=20, deleteDelay=180):
687687

688688
session = self.dbSession(expire_on_commit=False)
689689

690+
fromDate = datetime.datetime.utcnow() - datetime.timedelta(days=deleteDelay)
690691
try:
691692
ftsOps = (
692693
session.query(FTS3Operation.operationID)
693-
.filter(
694-
FTS3Operation.lastUpdate < (func.date_sub(utc_timestamp(), text("INTERVAL %d DAY" % deleteDelay)))
695-
)
694+
.filter(FTS3Operation.lastUpdate < fromDate)
696695
.filter(FTS3Operation.status.in_(FTS3Operation.FINAL_STATES))
697696
.limit(limit)
698697
)

src/DIRAC/DataManagementSystem/DB/test/FTS3TestUtils.py

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -342,5 +342,88 @@ def base_test_job_monitoring_solve_racecondition(fts3db, fts3Client):
342342
assert not filesToSubmit
343343

344344

345+
def base_test_delete_operations(fts3db, fts3Client):
346+
"""Test operation removals"""
347+
op1 = generateOperation("Transfer", 2, ["Target1"])
348+
349+
res = fts3Client.persistOperation(op1)
350+
opID1 = res["Value"]
351+
352+
# Create two other operations, to test the limit feature
353+
op2 = generateOperation("Transfer", 2, ["Target2"])
354+
res = fts3Client.persistOperation(op2)
355+
opID2 = res["Value"]
356+
357+
op3 = generateOperation("Transfer", 2, ["Target3"])
358+
res = fts3Client.persistOperation(op3)
359+
opID3 = res["Value"]
360+
361+
# Now, call delete, and make sure that operation is not delete
362+
# Ops is not in a final state, and delay is not passed
363+
res = fts3db.deleteFinalOperations()
364+
assert res["OK"]
365+
366+
res = fts3Client.getOperation(opID1)
367+
assert res["OK"]
368+
op1 = res["Value"]
369+
370+
# Try again with no delay, but still not final state
371+
res = fts3db.deleteFinalOperations(deleteDelay=0)
372+
assert res["OK"]
373+
374+
res = fts3Client.getOperation(opID1)
375+
assert res["OK"]
376+
op1 = res["Value"]
377+
378+
# Set the final status
379+
op1.status = "Finished"
380+
res = fts3Client.persistOperation(op1)
381+
assert res["OK"]
382+
383+
# Now try to delete again.
384+
# It should still not work because of the delay
385+
res = fts3db.deleteFinalOperations()
386+
assert res["OK"]
387+
388+
res = fts3Client.getOperation(opID1)
389+
assert res["OK"]
390+
op1 = res["Value"]
391+
392+
# Finally, it should work, with no delay and a final status
393+
res = fts3db.deleteFinalOperations(deleteDelay=0)
394+
assert res["OK"]
395+
396+
res = fts3Client.getOperation(opID1)
397+
assert not res["OK"]
398+
399+
# op2 and op3 should still be here though !
400+
401+
res = fts3Client.getOperation(opID2)
402+
assert res["OK"]
403+
op2 = res["Value"]
404+
res = fts3Client.getOperation(opID3)
405+
assert res["OK"]
406+
op3 = res["Value"]
407+
408+
# Set them both to a final status
409+
op2.status = "Finished"
410+
res = fts3Client.persistOperation(op2)
411+
assert res["OK"]
412+
op3.status = "Finished"
413+
res = fts3Client.persistOperation(op3)
414+
assert res["OK"]
415+
416+
# Now try to delete, but only one
417+
res = fts3db.deleteFinalOperations(limit=1, deleteDelay=0)
418+
assert res["OK"]
419+
420+
# Now only op2 or op3 should be here
421+
422+
res2 = fts3Client.getOperation(opID2)
423+
res3 = fts3Client.getOperation(opID3)
424+
425+
assert res2["OK"] ^ res3["OK"]
426+
427+
345428
# All base tests defined in this module
346429
allBaseTests = [test_func for testName, test_func in globals().items() if testName.startswith("base_test")]

testSQL.py

Lines changed: 0 additions & 170 deletions
This file was deleted.

0 commit comments

Comments
 (0)