From 74744aac4282c19de246a5b90ca12d1daca20bcc Mon Sep 17 00:00:00 2001 From: Toni Harzendorf Date: Sat, 25 Jan 2025 13:42:23 +0100 Subject: [PATCH 1/2] fix compilation with cython 0.29.37 --- pyslurm/core/partition.pxd | 3 +-- pyslurm/core/partition.pyx | 4 ++-- pyslurm/core/slurmctld/__init__.pxd | 4 ---- 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/pyslurm/core/partition.pxd b/pyslurm/core/partition.pxd index ee178230..2a877a1f 100644 --- a/pyslurm/core/partition.pxd +++ b/pyslurm/core/partition.pxd @@ -56,7 +56,6 @@ from pyslurm.utils cimport cstr from pyslurm.utils cimport ctime from pyslurm.utils.ctime cimport time_t from pyslurm.utils.uint cimport * -from pyslurm.core cimport slurmctld from pyslurm.xcollections cimport MultiClusterMap @@ -218,7 +217,7 @@ cdef class Partition: cdef: partition_info_t *ptr int power_save_enabled - slurmctld.Config slurm_conf + slurm_conf cdef readonly cluster diff --git a/pyslurm/core/partition.pyx b/pyslurm/core/partition.pyx index 53e41648..f29b9aab 100644 --- a/pyslurm/core/partition.pyx +++ b/pyslurm/core/partition.pyx @@ -31,6 +31,7 @@ from pyslurm.core.error import RPCError, verify_rpc from pyslurm.utils.ctime import timestamp_to_date, _raw_time from pyslurm.constants import UNLIMITED from pyslurm.settings import LOCAL_CLUSTER +from pyslurm.core import slurmctld from pyslurm.core.slurmctld.config import _get_memory from pyslurm import xcollections from pyslurm.utils.helpers import ( @@ -78,7 +79,6 @@ cdef class Partitions(MultiClusterMap): Partitions partitions = Partitions() int flags = slurm.SHOW_ALL Partition partition - slurmctld.Config slurm_conf int power_save_enabled = 0 verify_rpc(slurm_load_partitions(0, &partitions.info, flags)) @@ -801,7 +801,7 @@ def _preempt_mode_str_to_int(mode): return pmode -def _preempt_mode_int_to_str(mode, slurmctld.Config slurm_conf): +def _preempt_mode_int_to_str(mode, slurm_conf): if mode == slurm.NO_VAL16: return slurm_conf.preempt_mode if slurm_conf else None else: diff --git a/pyslurm/core/slurmctld/__init__.pxd b/pyslurm/core/slurmctld/__init__.pxd index edc6a267..e69de29b 100644 --- a/pyslurm/core/slurmctld/__init__.pxd +++ b/pyslurm/core/slurmctld/__init__.pxd @@ -1,4 +0,0 @@ -# cython: c_string_type=unicode, c_string_encoding=default -# cython: language_level=3 - -from .config cimport Config From cb4eab6f9d9bbce8659ed32dd399b38219ae3784 Mon Sep 17 00:00:00 2001 From: Toni Harzendorf Date: Sun, 26 Jan 2025 13:04:56 +0100 Subject: [PATCH 2/2] do not import LOCAL_CLUSTER directly, access it via settings module --- pyslurm/__init__.py | 8 ++++---- pyslurm/api.pyx | 3 +++ pyslurm/core/job/job.pyx | 4 ++-- pyslurm/core/job/step.pyx | 4 ++-- pyslurm/core/node.pyx | 6 +++--- pyslurm/core/partition.pyx | 6 +++--- pyslurm/db/assoc.pyx | 8 ++++---- pyslurm/db/job.pyx | 10 ++++------ pyslurm/settings.pyx | 16 +++++++++++----- pyslurm/xcollections.pxd | 1 + pyslurm/xcollections.pyx | 19 ++++++++++--------- 11 files changed, 47 insertions(+), 38 deletions(-) diff --git a/pyslurm/__init__.py b/pyslurm/__init__.py index 641faef7..c6e41eb7 100644 --- a/pyslurm/__init__.py +++ b/pyslurm/__init__.py @@ -7,10 +7,6 @@ sys.setdlopenflags(sys.getdlopenflags() | os.RTLD_GLOBAL | os.RTLD_DEEPBIND) -# Initialize slurm api -from pyslurm.api import slurm_init, slurm_fini -slurm_init() - from .version import __version__ from pyslurm import db @@ -35,3 +31,7 @@ # The old API in deprecated.pyx from pyslurm.deprecated import * + +# Initialize slurm api +from pyslurm.api import slurm_init, slurm_fini +slurm_init() diff --git a/pyslurm/api.pyx b/pyslurm/api.pyx index 0f34fedb..eb8a773c 100644 --- a/pyslurm/api.pyx +++ b/pyslurm/api.pyx @@ -22,6 +22,8 @@ # cython: c_string_type=unicode, c_string_encoding=default # cython: language_level=3 +from pyslurm import settings + def slurm_init(config_path=None): """Initialize the Slurm API. @@ -36,6 +38,7 @@ def slurm_init(config_path=None): None, so libslurm will automatically detect its config. """ slurm.slurm_init(cstr.from_unicode(config_path)) + settings.init() def slurm_fini(): diff --git a/pyslurm/core/job/job.pyx b/pyslurm/core/job/job.pyx index 9605d0f7..ad228e12 100644 --- a/pyslurm/core/job/job.pyx +++ b/pyslurm/core/job/job.pyx @@ -34,7 +34,7 @@ from typing import Union from pyslurm.utils import cstr, ctime from pyslurm.utils.uint import * from pyslurm.core.job.util import * -from pyslurm.settings import LOCAL_CLUSTER +from pyslurm import settings from pyslurm import xcollections from pyslurm.core.error import ( RPCError, @@ -236,7 +236,7 @@ cdef class Job: self.ptr.job_id = job_id self.passwd = {} self.groups = {} - cstr.fmalloc(&self.ptr.cluster, LOCAL_CLUSTER) + cstr.fmalloc(&self.ptr.cluster, settings.LOCAL_CLUSTER) self.steps = JobSteps() self.stats = JobStatistics() self.pids = {} diff --git a/pyslurm/core/job/step.pyx b/pyslurm/core/job/step.pyx index b3a09509..ad6a8e9c 100644 --- a/pyslurm/core/job/step.pyx +++ b/pyslurm/core/job/step.pyx @@ -26,7 +26,7 @@ from typing import Union from pyslurm.utils import cstr, ctime from pyslurm.utils.uint import * from pyslurm.core.error import RPCError, verify_rpc -from pyslurm.settings import LOCAL_CLUSTER +from pyslurm import settings from pyslurm import xcollections from pyslurm.utils.helpers import ( signal_to_num, @@ -159,7 +159,7 @@ cdef class JobStep: self.id = step_id self.stats = JobStepStatistics() self.pids = {} - cstr.fmalloc(&self.ptr.cluster, LOCAL_CLUSTER) + cstr.fmalloc(&self.ptr.cluster, settings.LOCAL_CLUSTER) # Initialize attributes, if any were provided for k, v in kwargs.items(): diff --git a/pyslurm/core/node.pyx b/pyslurm/core/node.pyx index b3cb5593..01489436 100644 --- a/pyslurm/core/node.pyx +++ b/pyslurm/core/node.pyx @@ -28,7 +28,7 @@ from pyslurm.utils import ctime from pyslurm.utils.uint import * from pyslurm.core.error import RPCError, verify_rpc from pyslurm.utils.ctime import timestamp_to_date, _raw_time -from pyslurm.settings import LOCAL_CLUSTER +from pyslurm import settings from pyslurm import xcollections from pyslurm.utils.helpers import ( uid_to_name, @@ -222,7 +222,7 @@ cdef class Node: def __init__(self, name=None, **kwargs): self._alloc_impl() self.name = name - self.cluster = LOCAL_CLUSTER + self.cluster = settings.LOCAL_CLUSTER for k, v in kwargs.items(): setattr(self, k, v) @@ -270,7 +270,7 @@ cdef class Node: wrap._alloc_info() wrap.passwd = {} wrap.groups = {} - wrap.cluster = LOCAL_CLUSTER + wrap.cluster = settings.LOCAL_CLUSTER memcpy(wrap.info, in_ptr, sizeof(node_info_t)) return wrap diff --git a/pyslurm/core/partition.pyx b/pyslurm/core/partition.pyx index f29b9aab..e54bc37b 100644 --- a/pyslurm/core/partition.pyx +++ b/pyslurm/core/partition.pyx @@ -30,7 +30,7 @@ from pyslurm.utils.uint import * from pyslurm.core.error import RPCError, verify_rpc from pyslurm.utils.ctime import timestamp_to_date, _raw_time from pyslurm.constants import UNLIMITED -from pyslurm.settings import LOCAL_CLUSTER +from pyslurm import settings from pyslurm.core import slurmctld from pyslurm.core.slurmctld.config import _get_memory from pyslurm import xcollections @@ -168,7 +168,7 @@ cdef class Partition: def __init__(self, name=None, **kwargs): self._alloc_impl() self.name = name - self.cluster = LOCAL_CLUSTER + self.cluster = settings.LOCAL_CLUSTER for k, v in kwargs.items(): setattr(self, k, v) @@ -194,7 +194,7 @@ cdef class Partition: cdef Partition from_ptr(partition_info_t *in_ptr): cdef Partition wrap = Partition.__new__(Partition) wrap._alloc_impl() - wrap.cluster = LOCAL_CLUSTER + wrap.cluster = settings.LOCAL_CLUSTER memcpy(wrap.ptr, in_ptr, sizeof(partition_info_t)) return wrap diff --git a/pyslurm/db/assoc.pyx b/pyslurm/db/assoc.pyx index 93617669..36460cbe 100644 --- a/pyslurm/db/assoc.pyx +++ b/pyslurm/db/assoc.pyx @@ -29,7 +29,7 @@ from pyslurm.utils.helpers import ( ) from pyslurm.utils.uint import * from pyslurm.db.connection import _open_conn_or_error -from pyslurm.settings import LOCAL_CLUSTER +from pyslurm import settings from pyslurm import xcollections @@ -65,7 +65,7 @@ cdef class Associations(MultiClusterMap): # Fetch Assoc Data assoc_data = SlurmList.wrap(slurmdb_associations_get( conn.ptr, cond.ptr)) - + if assoc_data.is_null: raise RPCError(msg="Failed to get Association data from slurmdbd") @@ -135,7 +135,7 @@ cdef class Associations(MultiClusterMap): else: # Autodetects the last slurm error raise RPCError() - + if not db_connection: # Autocommit if no connection was explicitly specified. conn.commit() @@ -185,7 +185,7 @@ cdef class Association: def __init__(self, **kwargs): self._alloc_impl() self.id = 0 - self.cluster = LOCAL_CLUSTER + self.cluster = settings.LOCAL_CLUSTER for k, v in kwargs.items(): setattr(self, k, v) diff --git a/pyslurm/db/job.pyx b/pyslurm/db/job.pyx index 0f4a0e9b..b6af6f91 100644 --- a/pyslurm/db/job.pyx +++ b/pyslurm/db/job.pyx @@ -24,10 +24,8 @@ from typing import Union, Any from pyslurm.core.error import RPCError, PyslurmError -from pyslurm.core import slurmctld -from typing import Any from pyslurm.utils.uint import * -from pyslurm.settings import LOCAL_CLUSTER +from pyslurm import settings from pyslurm import xcollections from pyslurm.utils.ctime import ( date_to_timestamp, @@ -106,7 +104,7 @@ cdef class JobFilter: if not self.clusters: # This is a requirement for some other parameters to function # correctly, like self.nodelist - return [LOCAL_CLUSTER] + return [settings.LOCAL_CLUSTER] elif self.clusters == "all": return None else: @@ -450,7 +448,7 @@ cdef class Job: self._alloc_impl() self.ptr.jobid = int(job_id) cstr.fmalloc(&self.ptr.cluster, - LOCAL_CLUSTER if not cluster else cluster) + settings.LOCAL_CLUSTER if not cluster else cluster) self.qos_data = QualitiesOfService() self.steps = JobSteps() self.stats = JobStatistics() @@ -511,7 +509,7 @@ cdef class Job: >>> db_job = pyslurm.db.Job.load(10000, with_script=True) >>> print(db_job.script) """ - cluster = LOCAL_CLUSTER if not cluster else cluster + cluster = settings.LOCAL_CLUSTER if not cluster else cluster jfilter = JobFilter(ids=[int(job_id)], clusters=[cluster], with_script=with_script, with_env=with_env) job = Jobs.load(jfilter).get((cluster, int(job_id))) diff --git a/pyslurm/settings.pyx b/pyslurm/settings.pyx index 36b834c2..3c44bc87 100644 --- a/pyslurm/settings.pyx +++ b/pyslurm/settings.pyx @@ -22,12 +22,18 @@ # cython: c_string_type=unicode, c_string_encoding=default # cython: language_level=3 -from pyslurm.core import slurmctld from pyslurm cimport slurm from pyslurm.utils cimport cstr -LOCAL_CLUSTER = cstr.to_unicode(slurm.slurm_conf.cluster_name) -if not LOCAL_CLUSTER: - slurm_conf = slurmctld.Config.load() - LOCAL_CLUSTER = slurm_conf.cluster_name +LOCAL_CLUSTER = "UNKNOWN" + + +def init(): + from pyslurm.core import slurmctld + + global LOCAL_CLUSTER + LOCAL_CLUSTER = cstr.to_unicode(slurm.slurm_conf.cluster_name) + if not LOCAL_CLUSTER: + slurm_conf = slurmctld.Config.load() + LOCAL_CLUSTER = slurm_conf.cluster_name diff --git a/pyslurm/xcollections.pxd b/pyslurm/xcollections.pxd index 98dfa713..57b5fb21 100644 --- a/pyslurm/xcollections.pxd +++ b/pyslurm/xcollections.pxd @@ -95,3 +95,4 @@ cdef class MultiClusterMap: _key_type _val_type _id_attr + _cluster diff --git a/pyslurm/xcollections.pyx b/pyslurm/xcollections.pyx index 4e99eaf5..57c14445 100644 --- a/pyslurm/xcollections.pyx +++ b/pyslurm/xcollections.pyx @@ -23,7 +23,7 @@ # cython: language_level=3 """Custom Collection utilities""" -from pyslurm.settings import LOCAL_CLUSTER +from pyslurm import settings import json from typing import Union, Any @@ -193,6 +193,7 @@ cdef class MultiClusterMap: self._key_type = key_type self._val_type = val_type self._id_attr = id_attr + self._cluster = settings.LOCAL_CLUSTER if init_data: self._init_data(data) @@ -201,15 +202,15 @@ cdef class MultiClusterMap: for item in data: if isinstance(item, self._key_type): item = self._val_type(item) - if LOCAL_CLUSTER not in self.data: - self.data[LOCAL_CLUSTER] = {} + if self._cluster not in self.data: + self.data[self._cluster] = {} - self.data[LOCAL_CLUSTER].update({self._item_id(item): item}) + self.data[self._cluster].update({self._item_id(item): item}) elif isinstance(data, str): itemlist = data.split(",") items = {self._key_type(item):self._val_type(item) for item in itemlist} - self.data[LOCAL_CLUSTER] = items + self.data[self._cluster] = items elif isinstance(data, dict): self.update(data) elif data is not None: @@ -223,8 +224,8 @@ cdef class MultiClusterMap: def _get_cluster(self): cluster = None - if not self.data or LOCAL_CLUSTER in self.data: - cluster = LOCAL_CLUSTER + if not self.data or self._cluster in self.data: + cluster = self._cluster else: try: cluster = next(iter(self.keys())) @@ -259,7 +260,7 @@ cdef class MultiClusterMap: try: cluster = self._get_cluster() except KeyError: - cluster = LOCAL_CLUSTER + cluster = self._cluster if not cluster in self.data: self.data[cluster] = {} @@ -394,7 +395,7 @@ cdef class MultiClusterMap: according to `next(iter(self.keys()))` will be used. Examples: - Get a Job from the LOCAL_CLUSTER + Get a Job from the local Cluster. >>> job_id = 1 >>> job = data.get(job_id)