Skip to content

[v9.0] feat (gfal2): put a timeout around stat calls #8332

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 1 commit into
base: integration
Choose a base branch
from
Draft

[v9.0] feat (gfal2): put a timeout around stat calls #8332

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
70 changes: 25 additions & 45 deletions src/DIRAC/Resources/Storage/GFAL2_StorageBase.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,9 @@
MAX_SINGLE_STREAM_SIZE = 1024 * 1024 * 10 # 10MB
MIN_BANDWIDTH = 0.5 * (1024 * 1024) # 0.5 MB/s

# Timeout when performing a stat call
STAT_TIMEOUT = 10


@contextmanager
def setGfalSetting(
Expand Down Expand Up @@ -218,11 +221,12 @@ def exists(self, path):
successful = {}
failed = {}

for url in urls:
try:
successful[url] = self.__singleExists(url)
except Exception as e:
failed[url] = repr(e)
with setGfalSetting(self.ctx, "CORE", "NAMESPACE_TIMEOUT", STAT_TIMEOUT):
for url in urls:
try:
successful[url] = self.__singleExists(url)
except Exception as e:
failed[url] = repr(e)

resDict = {"Failed": failed, "Successful": successful}
return resDict
Expand Down Expand Up @@ -268,29 +272,15 @@ def isFile(self, path):
successful = {}
failed = {}

for url in urls:
try:
successful[url] = self._isSingleFile(url)
except Exception as e:
failed[url] = repr(e)
with setGfalSetting(self.ctx, "CORE", "NAMESPACE_TIMEOUT", STAT_TIMEOUT):
for url in urls:
try:
successful[url] = S_ISREG(self.ctx.stat(url).st_mode)
except Exception as e:
failed[url] = repr(e)

return {"Failed": failed, "Successful": successful}

def _isSingleFile(self, path: str) -> bool:
"""Checking if :path: exists and is a file

:param str path: single path on the storage (srm://...)

:returns: boolean

:raises:
gfal2.GError: gfal problem

"""

statInfo = self.ctx.stat(path)
return S_ISREG(statInfo.st_mode)

@convertToReturnValue
def putFile(self, path, sourceSize: int = 0):
"""Put a copy of a local file or a file on another srm storage to a directory on the
Expand Down Expand Up @@ -600,7 +590,8 @@ def _getSingleFileSize(self, path: str) -> int:
log = self.log.getLocalSubLogger("GFAL2_StorageBase._getSingleFileSize")
log.debug(f"Determining file size of {path}")

statInfo = self.ctx.stat(path) # keeps info like size, mode.
with setGfalSetting(self.ctx, "CORE", "NAMESPACE_TIMEOUT", STAT_TIMEOUT):
statInfo = self.ctx.stat(path) # keeps info like size, mode.

# If it is not a file
if not S_ISREG(statInfo.st_mode):
Expand Down Expand Up @@ -675,7 +666,8 @@ def _getSingleMetadata(self, path: str) -> dict[str, Any]:
log = self.log.getLocalSubLogger("GFAL2_StorageBase._getSingleMetadata")
log.debug(f"Reading metadata for {path}")

statInfo = self.ctx.stat(path)
with setGfalSetting(self.ctx, "CORE", "NAMESPACE_TIMEOUT", STAT_TIMEOUT):
statInfo = self.ctx.stat(path)

metadataDict = self.__parseStatInfoFromApiOutput(statInfo)
if metadataDict["File"] and self.checksumType:
Expand Down Expand Up @@ -942,27 +934,15 @@ def isDirectory(self, path):
successful = {}
failed = {}

for url in urls:
try:
successful[url] = self._isSingleDirectory(url)
except Exception as e:
failed[url] = f"Failed to determine if path is a directory {repr(e)}"
with setGfalSetting(self.ctx, "CORE", "NAMESPACE_TIMEOUT", STAT_TIMEOUT):
for url in urls:
try:
successful[url] = S_ISDIR(self.ctx.stat(url).st_mode)
except Exception as e:
failed[url] = f"Failed to determine if path is a directory {repr(e)}"

return {"Failed": failed, "Successful": successful}

def _isSingleDirectory(self, path: str) -> bool:
"""Checking if :path: exists and is a directory

:param str path: single path on the storage (srm://...)

:returns: boolean if it is a directory or not

:raises:
gfal2.GError in case of gfal problem
"""

return S_ISDIR(self.ctx.stat(path).st_mode)

@convertToReturnValue
def listDirectory(self, path):
"""List the content of the path provided
Expand Down
Loading