WIP: support resource.reserve configuration parameter

doc/man5/flux-config-resource.rst

@@ -79,6 +79,21 @@ rediscover
    (optional) If true, force rediscovery of resources using HWLOC, rather
    then using the R and HWLOC XML from the enclosing instance.
 
+reserve
+   (optional) A string value that defines cores to reserve for the OS
+   and the broker ranks on which to reserve them. The argument is specified
+   as::
+
+    cores[@ranks] [cores[@ranks]]...
+
+   where cores is an RFC 22 idset specifying the cores to reserve, and
+   the optional ranks is an RFC 22 idset specifying the ranks on which to
+   reserve them. If ``@ranks`` is not supplied, then cores will be reserved
+   on all ranks. Multiple instances of ``cores[@ranks]`` may be specified
+   (separated by whitespace) to reserve a different set of cores on different
+   ranks. For example, ``0-3@0 0`` would reserve cores 0-3 on rank 0 and
+   core 0 on all other ranks.
+
 Note that updates to the resource table are ignored until the next Flux
 restart.
 

src/cmd/flux-resource.py

@@ -74,7 +74,7 @@ class FluxResourceConfig(UtilConfig):
             "description": "Format including resource list details",
             "format": (
                 "{state:>10} ?+:{queue:<5} ?:{propertiesx:<10.10+} {nnodes:>6} "
-                "+:{ncores:>6} ?:+{ngpus:>5} {rlist}"
+                "+:{ncores:>6} ?+:{ngpus:>5} {rlist}"
             ),
         },
     }

src/common/librlist/Makefile.am

@@ -25,6 +25,7 @@ librlist_la_SOURCES = \
 	match.c \
 	rlist.c \
 	rlist.h \
+	corespec.c \
 	rlist_private.h
 
 librlist_hwloc_la_SOURCES = \

src/common/librlist/corespec.c

@@ -0,0 +1,187 @@
+/************************************************************\
+ * Copyright 2025 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <errno.h>
+#include <string.h>
+
+#include <flux/core.h>
+#include <flux/idset.h>
+
+#include "src/common/libczmqcontainers/czmq_containers.h"
+#include "src/common/libutil/errprintf.h"
+
+#include "rlist.h"
+#include "rnode.h"
+#include "rlist_private.h"
+
+struct core_spec {
+    char *spec;
+    struct idset *cores;
+    struct idset *ranks;
+};
+
+static void core_spec_destroy (struct core_spec *spec)
+{
+    if (spec) {
+        int saved_errno = errno;
+        idset_destroy (spec->cores);
+        idset_destroy (spec->ranks);
+        free (spec->spec);
+        free (spec);
+        errno = saved_errno;
+    }
+}
+
+static struct core_spec *core_spec_create (const char *s, flux_error_t *errp)
+{
+    idset_error_t error;
+    struct core_spec *spec;
+    const char *ranks = NULL;
+    const char *cores;
+    int len = -1;
+
+    cores = s;
+    if ((ranks = strchr (cores, '@'))) {
+        len = ranks - cores;
+        ranks++;
+    }
+    if (!(spec = calloc (1, sizeof (*spec)))
+        || !(spec->spec = strdup (s))) {
+        errprintf (errp, "Out of memory");
+        goto error;
+    }
+    if ((ranks && !(spec->ranks = idset_decode_ex (ranks, -1, 0, 0, &error)))
+        || !(spec->cores = idset_decode_ex (cores, len, 0, 0, &error))) {
+        errprintf (errp, "%s", error.text);
+        goto error;
+    }
+    if ((spec->ranks && idset_count (spec->ranks) == 0)
+        || idset_count (spec->cores) == 0) {
+        errprintf (errp, "ranks/cores cannot be empty");
+        goto error;
+    }
+    return spec;
+error:
+    core_spec_destroy (spec);
+    return NULL;
+}
+
+static struct rnode *core_spec_copy (const struct rnode *orig,
+                                     struct core_spec *spec)
+{
+    struct rnode *n = NULL;
+
+    /* If spec->ranks is NULL, this indicates all ranks
+     */
+    if (!spec->ranks || idset_test (spec->ranks, orig->rank)) {
+        /* Create new rnode object with just the cores intersection, keeping
+         * hostname and properties.
+         */
+        struct idset *ids = idset_intersect (orig->cores->ids, spec->cores);
+        if (ids != NULL
+            && idset_count (ids) > 0
+            && (n = rnode_create_idset (orig->hostname, orig->rank, ids))) {
+            n->properties = zhashx_dup (orig->properties);
+        }
+        idset_destroy (ids);
+    }
+    return n;
+}
+
+static void core_spec_destructor (void **item)
+{
+    if (item) {
+        struct core_spec *spec = *item;
+        core_spec_destroy (spec);
+        *item = NULL;
+    }
+}
+
+static zlistx_t *core_spec_list_create (const char *core_spec,
+                                        flux_error_t *errp)
+{
+    char *copy;
+    char *str;
+    char *spec;
+    char *sp = NULL;
+    zlistx_t *l = zlistx_new ();
+
+    if (!l || !(copy = strdup (core_spec)))
+        return NULL;
+    str = copy;
+
+    zlistx_set_destructor (l, core_spec_destructor);
+
+    while ((spec = strtok_r (str, " \t", &sp))) {
+        struct core_spec *cspec;
+        if (!(cspec = core_spec_create (spec, errp)))
+            goto error;
+        if (!zlistx_add_end (l, cspec)) {
+            errprintf (errp, "Out of memory");
+            goto error;
+        }
+        str = NULL;
+    }
+    free (copy);
+    return l;
+error:
+    free (copy);
+    zlistx_destroy (&l);
+    return NULL;
+}
+
+struct rlist *rlist_copy_core_spec (const struct rlist *orig,
+                                    const char *core_spec,
+                                    flux_error_t *errp)
+{
+    struct core_spec *spec;
+    struct rlist *rl = NULL;
+    zlistx_t *l;
+
+    if (!(l = core_spec_list_create (core_spec, errp)))
+        return NULL;
+
+    spec = zlistx_first (l);
+    while (spec) {
+        struct rlist *tmp;
+        if (!(tmp = rlist_copy_internal (orig,
+                                         (rnode_copy_f) core_spec_copy,
+                                         (void *) spec))) {
+            errprintf (errp, "failed to copy spec '%s'", spec->spec);
+            goto error;
+        }
+        if (rl != NULL) {
+            if (rlist_add (rl, tmp) < 0) {
+                errprintf (errp,
+                           "rlist_add '%s' failed: %s",
+                           spec->spec,
+                           strerror (errno));
+                goto error;
+            }
+            rlist_destroy (tmp);
+        }
+        else
+            rl = tmp;
+        spec = zlistx_next (l);
+    }
+    zlistx_destroy (&l);
+    return rl;
+error:
+    rlist_destroy (rl);
+    zlistx_destroy (&l);
+    return NULL;
+}
+
+/* vi: ts=4 sw=4 expandtab
+ */

src/common/librlist/rlist.c

@@ -214,11 +214,9 @@
     return 0;
 }
 
-typedef struct rnode * (*rnode_copy_f) (const struct rnode *, void *arg);
-
-static struct rlist *rlist_copy_internal (const struct rlist *orig,
-                                          rnode_copy_f cpfn,
-                                          void *arg)
+struct rlist *rlist_copy_internal (const struct rlist *orig,
+                                   rnode_copy_f cpfn,
+                                   void *arg)
 {
     struct rnode *n;
     struct rlist *rl = rlist_create ();
@@ -535,17 +533,9 @@
     return n;
 }
 
-struct rlist *rlist_diff (const struct rlist *rla, const struct rlist *rlb)
+int rlist_subtract (struct rlist *rl, const struct rlist *rlb)
 {
-    struct rnode *n;
-    struct rlist *rl = rlist_create ();
-
-    if (!rl || rlist_append (rl, rla) < 0) {
-        rlist_destroy (rl);
-        return NULL;
-    }
-
-    n = zlistx_first (rlb->nodes);
+    struct rnode *n = zlistx_first (rlb->nodes);
     while (n) {
         /*  Attempt to find and "detach" the rank which we're diffing.
          */
@@ -567,6 +557,18 @@
         }
         n = zlistx_next (rlb->nodes);
     }
+    return 0;
+}
+
+struct rlist *rlist_diff (const struct rlist *rla, const struct rlist *rlb)
+{
+    struct rlist *rl = rlist_create ();
+    if (!rl
+        || rlist_append (rl, rla) < 0
+        || rlist_subtract (rl, rlb) < 0) {
+        rlist_destroy (rl);
+        return NULL;
+    }
     return rl;
 }
 

src/common/librlist/rlist.h

@@ -151,6 +151,11 @@ int rlist_append (struct rlist *rl, const struct rlist *rl2);
  */
 int rlist_add (struct rlist *rl, const struct rlist *rl2);
 
+/*  Subtract resources in `rl2` from `rl`. It is not an error if
+ *   resources in rl2 are not present in `rl`.
+ */
+int rlist_subtract (struct rlist *rl, const struct rlist *rl2);
+
 /*  Return the set difference of 'rlb' from 'rla'.
  */
 struct rlist *rlist_diff (const struct rlist *rla, const struct rlist *rlb);
@@ -287,5 +292,17 @@ char *rlist_properties_encode (const struct rlist *rl);
 
 struct rlist *rlist_from_config (json_t *conf, flux_error_t *errp);
 
+/*  Create a copy of rlist 'rl' using 'core_spec', which indicates a
+ *  set of cores via the form:
+ *
+ *    cores[@ranks] [cores[@ranks]]...
+ *
+ *  Where 'cores' is an idset of cores to copy and the optional 'ranks'
+ *  and idset of ranks from which to copy them. It is not an error to
+ *  specify cores and or ranks that do not exist in the source rlist 'rl'.
+ */
+struct rlist *rlist_copy_core_spec (const struct rlist *rl,
+                                    const char *spec,
+                                    flux_error_t *errp);
 
 #endif /* !HAVE_SCHED_RLIST_H */

src/common/librlist/rlist_private.h

@@ -13,4 +13,10 @@
 
 int rlist_add_rnode (struct rlist *rl, struct rnode *n);
 
+typedef struct rnode * (*rnode_copy_f) (const struct rnode *, void *arg);
+
+struct rlist *rlist_copy_internal (const struct rlist *orig,
+                                   rnode_copy_f cpfn,
+                                   void *arg);
+
 #endif /* !HAVE_SCHED_RLIST_PRIVATE_H */