feat: add scitag support

Author: fstagni

src/DIRAC/DataManagementSystem/Client/FTS3Job.py

@@ -2,13 +2,15 @@
 
 import datetime
 import errno
-from packaging.version import Version
 
+import fts3.rest.client.easy as fts3
+import requests
+from cachetools import TTLCache, cached
 
 # Requires at least version 3.3.3
 from fts3 import __version__ as fts3_version
-import fts3.rest.client.easy as fts3
 from fts3.rest.client.exceptions import FTS3ClientException, NotFound
+from packaging.version import Version
 
 # There is a breaking change in the API in 3.13
 # https://gitlab.cern.ch/fts/fts-rest-flask/-/commit/5faa283e0cd4b80a0139a547c4a6356522c8449d
@@ -22,21 +24,56 @@
 # default pycurl. See https://its.cern.ch/jira/browse/FTS-261
 from fts3.rest.client.request import Request as ftsSSLRequest
 
-from DIRAC.Resources.Storage.StorageElement import StorageElement
-
-from DIRAC.FrameworkSystem.Client.Logger import gLogger
-from DIRAC.FrameworkSystem.Client.TokenManagerClient import gTokenManager
-
-from DIRAC.Core.Utilities.ReturnValues import S_OK, S_ERROR
 from DIRAC.Core.Utilities.DErrno import cmpError
-
 from DIRAC.Core.Utilities.JEncode import JSerializable
+from DIRAC.Core.Utilities.ReturnValues import S_ERROR, S_OK
 from DIRAC.DataManagementSystem.Client.FTS3File import FTS3File
+from DIRAC.FrameworkSystem.Client.Logger import gLogger
+from DIRAC.FrameworkSystem.Client.TokenManagerClient import gTokenManager
+from DIRAC.Resources.Storage.StorageElement import StorageElement
 
 # 3 days in seconds
 BRING_ONLINE_TIMEOUT = 259200
 
 
+def get_scitag(vo: str, activity: str = None):
+    """
+    Get the scitag based on the VO and activity.
+    If the VO is not found in the scitag.json, it defaults to 1.
+    If no specific activity is provided, it defaults to the "default" activityName.
+
+    :param vo: The VO for which to get the scitag
+    :param activity: The activity for which to get the scitag
+    :return: The scitag value
+    """
+
+    # Create a TTL cache: max 1 item, expires after 84400 seconds (1 day)
+    json_cache = TTLCache(maxsize=1, ttl=86400)
+
+    @cached(json_cache)
+    def get_remote_json():
+        gLogger.verbose("Fetching https://scitags.org/api.json from the network")
+        response = requests.get("https://scitags.org/api.json")
+        response.raise_for_status()
+        return response.json()
+
+    # Load the JSON data from the cache or network
+    sj = get_remote_json()
+
+    vo_id = 1  # Default VO ID
+    activity_id = 1  # Default activity ID
+
+    for experiment in sj.get("experiments", []):
+        if experiment.get("expName") == vo.lower():
+            vo_id = experiment.get("expId")
+            for act in experiment.get("activities", []):
+                if act.get("activityName") == activity:
+                    activity_id = act.get("activityId")
+
+    # Logic to determine the scitag based on vo and activity (this is what FTS wants)
+    return vo_id << 6 | activity_id  # Example logic, replace with actual implementation
+
+
 class FTS3Job(JSerializable):
     """Abstract class to represent a job to be executed by FTS. It belongs
     to an FTS3Operation
@@ -467,6 +504,9 @@ def _constructTransferJob(self, pinTime, allLFNs, target_spacetoken, protocols=N
                 # * stageProto://myFile -> stageProto://myFile
                 # * srcProto://myFile -> destProto://myFile
 
+                # scitag 65 is 1 << 6 | 1 (default experiment, default activity)
+                scitag = get_scitag(vo=self.vo, activity=self.activity)
+
                 if stageURL:
                     # We do not set a fileID in the metadata
                     # such that we do not update the DB when monitoring
@@ -485,6 +525,7 @@ def _constructTransferJob(self, pinTime, allLFNs, target_spacetoken, protocols=N
                         filesize=ftsFile.size,
                         metadata=stageTrans_metadata,
                         activity=self.activity,
+                        scitag=scitag,
                     )
                     transfers.append(stageTrans)
 
@@ -553,6 +594,7 @@ def _constructTransferJob(self, pinTime, allLFNs, target_spacetoken, protocols=N
                     activity=self.activity,
                     source_token=srcToken,
                     destination_token=dstToken,
+                    scitag=scitag,
                 )
 
                 transfers.append(trans)