Merge pull request #4566 from grondo/resource.config

resource: support configuration of resources in TOML config

Author: mergify[bot]

doc/man5/flux-config-resource.rst

@@ -24,6 +24,36 @@ path
    passed from the enclosing Flux instance.  The ``flux R`` utility may be
    used to generate this file.
 
+config
+   (optional) An array of resource config entries used as an alternative to
+   a R object configured by ``resource.path``. Each array entry must contain
+   a ``hosts`` key in RFC 29 Hostlist Format which configures the list of
+   hosts to which the rest of the entry applies. The entry may also contain
+   ``cores`` and/or ``gpus`` keys which configure the set of core ids and
+   GPU ids (in RFC 22 idset form) available on the targeted hosts, or a
+   ``properties`` key which is an array of property strings to assign to
+   ``hosts``. It is not an error to list a host multiple times, instead
+   each entry updates ``hosts``. If the ``config`` array exists, then any
+   ``path`` is ignored.
+
+   Example::
+
+     [[resource.config]]
+     hosts = "test[1-100]"
+     cores = "0-7"
+
+     [[resource.config]]
+     hosts = "test[1,2]"
+     gpus = "0-1"
+
+     [[resource.config]]
+     hosts = "test[1-89]"
+     properties = ["batch"]
+
+     [[resource.config]]
+     hosts = "test[90-100]"
+     properties = ["debug"]
+
 exclude
    (optional) A string value that defines one or more nodes to withhold
    from scheduling, either in RFC 22 idset form, or in RFC 29 hostlist form.

src/cmd/flux-R.c

@@ -29,6 +29,7 @@
 #include "src/common/libutil/read_all.h"
 #include "src/common/librlist/rlist.h"
 #include "src/common/librlist/rhwloc.h"
+#include "src/common/libtomlc99/toml.h"
 
 #define RSET_DOC "\
 Read, generate, and process RFC 20 Resource Set objects.\n\
@@ -43,6 +44,7 @@ int cmd_rerank (optparse_t *p, int argc, char **argv);
 int cmd_decode (optparse_t *p, int argc, char **argv);
 int cmd_verify (optparse_t *p, int argc, char **argv);
 int cmd_set_property (optparse_t *p, int argc, char **argv);
+int cmd_parse_config (optparse_t *p, int argc, char **argv);
 
 static struct optparse_option global_opts[] =  {
     OPTPARSE_TABLE_END
@@ -145,6 +147,11 @@ static struct optparse_option decode_opts[] = {
     OPTPARSE_TABLE_END
 };
 
+static struct optparse_option parse_config_opts[] = {
+    OPTPARSE_TABLE_END
+};
+
+
 static struct optparse_subcommand subcommands[] = {
     { "encode",
       "[OPTIONS]...",
@@ -218,6 +225,13 @@ static struct optparse_subcommand subcommands[] = {
       0,
       set_property_opts,
     },
+    { "parse-config",
+      "PATH",
+      "Read config from resource.config array",
+      cmd_parse_config,
+      0,
+      parse_config_opts,
+    },
     OPTPARSE_SUBCMD_END
 };
 
@@ -828,6 +842,33 @@ int cmd_set_property (optparse_t *p, int argc, char **argv)
     return 0;
 }
 
+int cmd_parse_config (optparse_t *p, int argc, char **argv)
+{
+    flux_error_t error;
+    json_t *o = NULL;
+    struct rlist *rl = NULL;
+    flux_conf_t *conf;
+
+    if (!(conf = flux_conf_parse (argv[1], &error)))
+        log_msg_exit ("flux_conf_parse: %s", error.text);
+
+    if (flux_conf_unpack (conf, &error,
+                          "{s:{s:o}}",
+                          "resource",
+                          "config", &o) < 0)
+        log_msg_exit ("Config file error: %s", error.text);
+
+    if (!(rl = rlist_from_config (o, &error)))
+        log_msg_exit ("Config file error: %s", error.text);
+
+    rlist_puts (rl);
+
+    flux_conf_decref (conf);
+    rlist_destroy (rl);
+
+    return 0;
+}
+
 /*
  * vi:tabstop=4 shiftwidth=4 expandtab
  */

src/common/librlist/rlist.c

@@ -2297,5 +2297,211 @@ struct rlist *rlist_from_hwloc (int rank, const char *xml)
     return NULL;
 }
 
+/*  Check if a resource set provided by configuration is valid.
+ *  Returns -1 on failure with error in errp->text.
+ */
+static int rlist_config_check (struct rlist *rl, flux_error_t *errp)
+{
+    struct rnode *n;
+    struct hostlist *empty;
+    int rc = -1;
+
+    if (zlistx_size (rl->nodes) == 0)
+        return errprintf (errp, "no hosts configured");
+
+    if (!(empty = hostlist_create ()))
+        return errprintf (errp, "hostlist_create: Out of memory");
+
+    n = zlistx_first (rl->nodes);
+    while (n) {
+        if (rnode_avail_total (n) <= 0) {
+            if (hostlist_append (empty, n->hostname) < 0) {
+                errprintf (errp,
+                           "host %s was assigned no resources",
+                           n->hostname);
+                goto out;
+            }
+        }
+        n = zlistx_next (rl->nodes);
+    }
+    if (hostlist_count (empty) > 0) {
+        char *s = hostlist_encode (empty);
+        errprintf (errp, "resource.config: %s assigned no resources", s);
+        free (s);
+        goto out;
+    }
+    rc = 0;
+out:
+    hostlist_destroy (empty);
+    return rc;
+}
+
+/*  Process one entry from the resource.config array
+ */
+static int rlist_config_add_entry (struct rlist *rl,
+                                   struct hostlist *hostmap,
+                                   flux_error_t *errp,
+                                   int index,
+                                   const char *hosts,
+                                   const char *cores,
+                                   const char *gpus,
+                                   json_t *properties)
+{
+    struct hostlist *hl = NULL;
+    const char *host = NULL;
+    struct idset *coreids = NULL;
+    struct idset *gpuids = NULL;
+    struct idset *ranks = NULL;
+    int rc = -1;
+
+    if (!(hl = hostlist_decode (hosts))) {
+        errprintf (errp, "config[%d]: invalid hostlist '%s'", index, hosts);
+        goto error;
+    }
+    if (hostlist_count (hl) == 0) {
+        errprintf (errp, "config[%d]: empty hostlist specified", index);
+        goto error;
+    }
+    if (!(ranks = idset_create (0, IDSET_FLAG_AUTOGROW))) {
+        errprintf (errp, "idset_create: %s", strerror (errno));
+        goto error;
+    }
+    if (cores && !(coreids = idset_decode (cores))) {
+        errprintf (errp, "config[%d]: invalid idset cores='%s'", index, cores);
+        goto error;
+    }
+    if (gpus && !(gpuids = idset_decode (gpus))) {
+        errprintf (errp, "config[%d]: invalid idset gpus='%s'", index, gpus);
+        goto error;
+    }
+    host = hostlist_first (hl);
+    while (host) {
+        struct rnode *n;
+        int rank = hostlist_find (hostmap, host);
+        if (rank < 0) {
+            /*
+             *  First time encountering this host. Append to host map
+             *   hostlist and assign a rank.
+             */
+            if (hostlist_append (hostmap, host) < 0) {
+                errprintf (errp, "failed to append %s to host map", host);
+                goto error;
+            }
+            rank = hostlist_count (hostmap) - 1;
+        }
+        if (idset_set (ranks, rank) < 0) {
+            errprintf (errp, "idset_set(ranks, %d): %s",
+                       rank,
+                       strerror (errno));
+            goto error;
+        }
+        if (!(n = rnode_new (host, rank))) {
+            errprintf (errp, "rnode_new: %s", strerror (errno));
+            goto error;
+        }
+        if (coreids && !rnode_add_child_idset (n, "core", coreids, coreids)) {
+            errprintf (errp, "rnode_add_child_idset: %s", strerror (errno));
+            goto error;
+        }
+        if (gpuids && !rnode_add_child_idset (n, "gpu", gpuids, gpuids)) {
+            errprintf (errp, "rnode_add_child_idset: %s", strerror (errno));
+            goto error;
+        }
+        if (properties) {
+            size_t idx;
+            json_t *o;
+
+            json_array_foreach (properties, idx, o) {
+                const char *property;
+                if (!(property = json_string_value (o))
+                    || property_string_invalid (property)) {
+                    char *s = json_dumps (o, JSON_ENCODE_ANY);
+                    errprintf (errp,
+                               "config[%d]: invalid property \"%s\"",
+                               index,
+                               s);
+                    free (s);
+                    goto error;
+                }
+                if (rnode_set_property (n, property) < 0) {
+                    errprintf (errp,
+                               "Failed to set property %s on rank %u",
+                                property,
+                                rank);
+                    goto error;
+                }
+            }
+        }
+        if (rlist_add_rnode (rl, n) < 0) {
+            errprintf (errp, "Unable to add rnode: %s", strerror (errno));
+            goto error;
+        }
+        host = hostlist_next (hl);
+    }
+    rc = 0;
+error:
+    hostlist_destroy (hl);
+    idset_destroy (ranks);
+    idset_destroy (coreids);
+    idset_destroy (gpuids);
+    return rc;
+}
+
+struct rlist *rlist_from_config (json_t *conf, flux_error_t *errp)
+{
+    size_t index;
+    json_t *entry;
+    struct rlist *rl = NULL;
+    struct hostlist *hl = NULL;
+
+    if (!conf || !json_is_array (conf)) {
+        errprintf (errp, "resource config must be an array");
+        return NULL;
+    }
+
+    if (!(hl = hostlist_create ())
+        || !(rl = rlist_create ())) {
+        errprintf (errp, "Out of memory");
+        goto error;
+    }
+
+    json_array_foreach (conf, index, entry) {
+        const char *hosts = NULL;
+        const char *cores = NULL;
+        const char *gpus = NULL;
+        json_t *properties = NULL;
+        json_error_t error;
+
+        if (json_unpack_ex (entry, &error, 0,
+                            "{s:s s?s s?s s?o !}",
+                            "hosts", &hosts,
+                            "cores", &cores,
+                            "gpus",  &gpus,
+                            "properties", &properties) < 0) {
+            errprintf (errp, "config[%ld]: %s", index, error.text);
+            goto error;
+        }
+        if (rlist_config_add_entry (rl,
+                                    hl,
+                                    errp,
+                                    index,
+                                    hosts,
+                                    cores,
+                                    gpus,
+                                    properties) < 0)
+            goto error;
+    }
+
+    if (rlist_config_check (rl, errp) < 0)
+        goto error;
+
+    hostlist_destroy (hl);
+    return rl;
+error:
+    hostlist_destroy (hl);
+    rlist_destroy (rl);
+    return NULL;
+}
+
 /* vi: ts=4 sw=4 expandtab
  */

src/common/librlist/rlist.h

@@ -280,4 +280,6 @@ int rlist_assign_properties (struct rlist *rl,
  */
 char *rlist_properties_encode (struct rlist *rl);
 
+struct rlist *rlist_from_config (json_t *conf, flux_error_t *errp);
+
 #endif /* !HAVE_SCHED_RLIST_H */

src/common/librlist/rnode.c

@@ -125,10 +125,10 @@ static int rnode_child_add_idset (struct rnode_child *c,
  *   then add 'ids' to that child (it is an error if one or more ids
  *   are already set in child 'name'.
  */
-static struct rnode_child * rnode_add_child_idset (struct rnode *n,
-                                                   const char *name,
-                                                   const struct idset *ids,
-                                                   const struct idset *avail)
+struct rnode_child * rnode_add_child_idset (struct rnode *n,
+                                            const char *name,
+                                            const struct idset *ids,
+                                            const struct idset *avail)
 {
     struct rnode_child *c = zhashx_lookup (n->children, name);
 
@@ -187,6 +187,7 @@ struct rnode *rnode_new (const char *name, uint32_t rank)
 
 int rnode_add (struct rnode *orig, struct rnode *n)
 {
+    int rc = 0;
     struct rnode_child *c;
     if (!orig || !n)
         return -1;
@@ -197,7 +198,21 @@ int rnode_add (struct rnode *orig, struct rnode *n)
             return -1;
         c = zhashx_next (n->children);
     }
-    return 0;
+    if (n->properties) {
+        zlistx_t *l = zhashx_keys (n->properties);
+        if (l != NULL) {
+            const char *property = zlistx_first (l);
+            while (property) {
+                if (rnode_set_property (orig, property) < 0)
+                    rc = -1;
+                property = zlistx_next (l);
+            }
+            zlistx_destroy (&l);
+        }
+        else
+            rc = -1;
+    }
+    return rc;
 }
 
 struct rnode *rnode_create (const char *name, uint32_t rank, const char *ids)

src/common/librlist/rnode.h

@@ -41,6 +41,13 @@ struct rnode {
     zhashx_t *properties;
 };
 
+struct rnode *rnode_new (const char *name, uint32_t rank);
+
+struct rnode_child * rnode_add_child_idset (struct rnode *n,
+                                            const char *name,
+                                            const struct idset *ids,
+                                            const struct idset *avail);
+
 /*  Create a resource node object from an existing idset `set`
  */
 struct rnode *rnode_create_idset (const char *name,