From e4a120c231ddf5213aac61a03e0d3a403244069e Mon Sep 17 00:00:00 2001 From: "Mark A. Grondona" Date: Thu, 9 Jan 2025 08:03:25 -0800 Subject: [PATCH 1/8] librlist: expose `rlist_copy_internal()` in rlist_private.h Problem: rlist_copy_internal() is declared static which means all copy implementations must be defined directly in rlist.c. Expose rlist_copy_internal() in rlist_private.h to allow copy implementations to be placed in separate source files. --- src/common/librlist/rlist.c | 8 +++----- src/common/librlist/rlist_private.h | 6 ++++++ 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/common/librlist/rlist.c b/src/common/librlist/rlist.c index a453618dff19..3232b6d00804 100644 --- a/src/common/librlist/rlist.c +++ b/src/common/librlist/rlist.c @@ -214,11 +214,9 @@ int rlist_add_rnode (struct rlist *rl, struct rnode *n) return 0; } -typedef struct rnode * (*rnode_copy_f) (const struct rnode *, void *arg); - -static struct rlist *rlist_copy_internal (const struct rlist *orig, - rnode_copy_f cpfn, - void *arg) +struct rlist *rlist_copy_internal (const struct rlist *orig, + rnode_copy_f cpfn, + void *arg) { struct rnode *n; struct rlist *rl = rlist_create (); diff --git a/src/common/librlist/rlist_private.h b/src/common/librlist/rlist_private.h index e3fa0fdc7724..b6d07cf3e07f 100644 --- a/src/common/librlist/rlist_private.h +++ b/src/common/librlist/rlist_private.h @@ -13,4 +13,10 @@ int rlist_add_rnode (struct rlist *rl, struct rnode *n); +typedef struct rnode * (*rnode_copy_f) (const struct rnode *, void *arg); + +struct rlist *rlist_copy_internal (const struct rlist *orig, + rnode_copy_f cpfn, + void *arg); + #endif /* !HAVE_SCHED_RLIST_PRIVATE_H */ From ec7bb0db394286fda1e13d81370b7b622967912d Mon Sep 17 00:00:00 2001 From: "Mark A. Grondona" Date: Wed, 8 Jan 2025 08:56:12 -0800 Subject: [PATCH 2/8] librlist: add rlist_copy_core_spec() Problem: There is no way to copy a set of cores from an rlist, but this would be useful when creating a set of cores to reserve for the OS. Add a new function rlist_copy_core_spec() which can copy cores from an rlist object using a specification of the form: cores[@ranks] [cores[@ranks]]... where `cores` and the optional `ranks` are idsets of the cores/ranks to include in the copy. --- src/common/librlist/Makefile.am | 1 + src/common/librlist/corespec.c | 187 ++++++++++++++++++++++++++++++++ src/common/librlist/rlist.h | 12 ++ 3 files changed, 200 insertions(+) create mode 100644 src/common/librlist/corespec.c diff --git a/src/common/librlist/Makefile.am b/src/common/librlist/Makefile.am index 9703294ff540..a4617c327f63 100644 --- a/src/common/librlist/Makefile.am +++ b/src/common/librlist/Makefile.am @@ -25,6 +25,7 @@ librlist_la_SOURCES = \ match.c \ rlist.c \ rlist.h \ + corespec.c \ rlist_private.h librlist_hwloc_la_SOURCES = \ diff --git a/src/common/librlist/corespec.c b/src/common/librlist/corespec.c new file mode 100644 index 000000000000..d4f8b9f29ee2 --- /dev/null +++ b/src/common/librlist/corespec.c @@ -0,0 +1,187 @@ +/************************************************************\ + * Copyright 2025 Lawrence Livermore National Security, LLC + * (c.f. AUTHORS, NOTICE.LLNS, COPYING) + * + * This file is part of the Flux resource manager framework. + * For details, see https://github.com/flux-framework. + * + * SPDX-License-Identifier: LGPL-3.0 +\************************************************************/ + +#if HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include + +#include +#include + +#include "src/common/libczmqcontainers/czmq_containers.h" +#include "src/common/libutil/errprintf.h" + +#include "rlist.h" +#include "rnode.h" +#include "rlist_private.h" + +struct core_spec { + char *spec; + struct idset *cores; + struct idset *ranks; +}; + +static void core_spec_destroy (struct core_spec *spec) +{ + if (spec) { + int saved_errno = errno; + idset_destroy (spec->cores); + idset_destroy (spec->ranks); + free (spec->spec); + free (spec); + errno = saved_errno; + } +} + +static struct core_spec *core_spec_create (const char *s, flux_error_t *errp) +{ + idset_error_t error; + struct core_spec *spec; + const char *ranks = NULL; + const char *cores; + int len = -1; + + cores = s; + if ((ranks = strchr (cores, '@'))) { + len = ranks - cores; + ranks++; + } + if (!(spec = calloc (1, sizeof (*spec))) + || !(spec->spec = strdup (s))) { + errprintf (errp, "Out of memory"); + goto error; + } + if ((ranks && !(spec->ranks = idset_decode_ex (ranks, -1, 0, 0, &error))) + || !(spec->cores = idset_decode_ex (cores, len, 0, 0, &error))) { + errprintf (errp, "%s", error.text); + goto error; + } + if ((spec->ranks && idset_count (spec->ranks) == 0) + || idset_count (spec->cores) == 0) { + errprintf (errp, "ranks/cores cannot be empty"); + goto error; + } + return spec; +error: + core_spec_destroy (spec); + return NULL; +} + +static struct rnode *core_spec_copy (const struct rnode *orig, + struct core_spec *spec) +{ + struct rnode *n = NULL; + + /* If spec->ranks is NULL, this indicates all ranks + */ + if (!spec->ranks || idset_test (spec->ranks, orig->rank)) { + /* Create new rnode object with just the cores intersection, keeping + * hostname and properties. + */ + struct idset *ids = idset_intersect (orig->cores->ids, spec->cores); + if (ids != NULL + && idset_count (ids) > 0 + && (n = rnode_create_idset (orig->hostname, orig->rank, ids))) { + n->properties = zhashx_dup (orig->properties); + } + idset_destroy (ids); + } + return n; +} + +static void core_spec_destructor (void **item) +{ + if (item) { + struct core_spec *spec = *item; + core_spec_destroy (spec); + *item = NULL; + } +} + +static zlistx_t *core_spec_list_create (const char *core_spec, + flux_error_t *errp) +{ + char *copy; + char *str; + char *spec; + char *sp = NULL; + zlistx_t *l = zlistx_new (); + + if (!l || !(copy = strdup (core_spec))) + return NULL; + str = copy; + + zlistx_set_destructor (l, core_spec_destructor); + + while ((spec = strtok_r (str, " \t", &sp))) { + struct core_spec *cspec; + if (!(cspec = core_spec_create (spec, errp))) + goto error; + if (!zlistx_add_end (l, cspec)) { + errprintf (errp, "Out of memory"); + goto error; + } + str = NULL; + } + free (copy); + return l; +error: + free (copy); + zlistx_destroy (&l); + return NULL; +} + +struct rlist *rlist_copy_core_spec (const struct rlist *orig, + const char *core_spec, + flux_error_t *errp) +{ + struct core_spec *spec; + struct rlist *rl = NULL; + zlistx_t *l; + + if (!(l = core_spec_list_create (core_spec, errp))) + return NULL; + + spec = zlistx_first (l); + while (spec) { + struct rlist *tmp; + if (!(tmp = rlist_copy_internal (orig, + (rnode_copy_f) core_spec_copy, + (void *) spec))) { + errprintf (errp, "failed to copy spec '%s'", spec->spec); + goto error; + } + if (rl != NULL) { + if (rlist_add (rl, tmp) < 0) { + errprintf (errp, + "rlist_add '%s' failed: %s", + spec->spec, + strerror (errno)); + goto error; + } + rlist_destroy (tmp); + } + else + rl = tmp; + spec = zlistx_next (l); + } + zlistx_destroy (&l); + return rl; +error: + rlist_destroy (rl); + zlistx_destroy (&l); + return NULL; +} + +/* vi: ts=4 sw=4 expandtab + */ diff --git a/src/common/librlist/rlist.h b/src/common/librlist/rlist.h index d2fc7d6b8ad3..9a1eb18c3901 100644 --- a/src/common/librlist/rlist.h +++ b/src/common/librlist/rlist.h @@ -287,5 +287,17 @@ char *rlist_properties_encode (const struct rlist *rl); struct rlist *rlist_from_config (json_t *conf, flux_error_t *errp); +/* Create a copy of rlist 'rl' using 'core_spec', which indicates a + * set of cores via the form: + * + * cores[@ranks] [cores[@ranks]]... + * + * Where 'cores' is an idset of cores to copy and the optional 'ranks' + * and idset of ranks from which to copy them. It is not an error to + * specify cores and or ranks that do not exist in the source rlist 'rl'. + */ +struct rlist *rlist_copy_core_spec (const struct rlist *rl, + const char *spec, + flux_error_t *errp); #endif /* !HAVE_SCHED_RLIST_H */ From eab24c3a950287907a5a3b3a54fc4ba23eae6dc3 Mon Sep 17 00:00:00 2001 From: "Mark A. Grondona" Date: Wed, 8 Jan 2025 10:37:51 -0800 Subject: [PATCH 3/8] librlist: add tests for rlist_copy_core_spec() Problem: There are no unit tests of the new function rlist_copy_core_spec(). Add unit tests. --- src/common/librlist/test/rlist.c | 69 ++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/src/common/librlist/test/rlist.c b/src/common/librlist/test/rlist.c index eebb7ec50bd2..b6217f7d4f8b 100644 --- a/src/common/librlist/test/rlist.c +++ b/src/common/librlist/test/rlist.c @@ -2149,6 +2149,74 @@ static void test_issue_5868 (void) free (R); } +struct core_spec_test { + const char *ranks; + const char *cores; + const char *hosts; + + const char *spec; + const char *result; + const char *error; +}; + +struct core_spec_test core_spec_tests[] = { + { "0-3", "0-3", "foo[0-3]", "0", "rank[0-3]/core0", NULL }, + { "0-3", "0-3", "foo[0-3]", "0-1", "rank[0-3]/core[0-1]", NULL }, + { "0-3", "0-3", "foo[0-3]", "0@0", "rank0/core0", NULL }, + { "0-3", "0-3", "foo[0-3]", "0,2@0", "rank0/core[0,2]", NULL }, + { "0-3", "0-3", "foo[0-3]", "0@0-1", "rank[0-1]/core0", NULL }, + { "0-3", "0-3", "foo[0-3]", "0-7@0", "rank0/core[0-3]", NULL }, + { "0-3", + "0-3", + "foo[0-3]", + "0-3@0 0@1-3", + "rank0/core[0-3] rank[1-3]/core0", + NULL }, + { "0-3", "0-3", "foo[0-3]", "foo", NULL, "error parsing range 'foo'"}, + { "0-3", "0-3", "foo[0-3]", "0@", NULL, "ranks/cores cannot be empty"}, + { "0-3", "0-3", "foo[0-3]", "@0", NULL, "ranks/cores cannot be empty"}, + { "0-3", "0-3", "foo[0-3]", "0 0@", NULL, "ranks/cores cannot be empty"}, + { NULL, NULL, NULL, NULL, NULL, NULL }, +}; + +static void test_core_spec (void) +{ + struct core_spec_test *te = &core_spec_tests[0]; + while (te && te->ranks) { + char *R; + struct rlist *rl; + struct rlist *result; + flux_error_t error; + + if (!(R = R_create (te->ranks, te->cores, NULL, te->hosts, NULL))) + BAIL_OUT ("test_core_spec: R_create"); + + if (!(rl = rlist_from_R (R))) + BAIL_OUT ("test_core_spec: rlist_from_R() failed"); + + result = rlist_copy_core_spec (rl, te->spec, &error); + if (result) { + char *s = rlist_dumps (result); + pass ("rlist_copy_core_spec (%s) returned %s", te->spec, s); + if (te->result) + is (s, te->result, "got expected result"); + else + fail ("got %s but expected failure", s); + free (s); + } + else if (te->error) { + pass ("rlist_copy_core_spec (%s) failed as expected", te->spec); + is (error.text, te->error, "got expected error: %s", error.text); + } + else + diag ("rlist_copy_core_spec (%s): %s", te->spec, error.text); + free (R); + rlist_destroy (rl); + rlist_destroy (result); + te++; + } +} + int main (int ac, char *av[]) { plan (NO_PLAN); @@ -2180,6 +2248,7 @@ int main (int ac, char *av[]) test_issue4290 (); test_rlist_config_inval (); test_issue_5868 (); + test_core_spec (); done_testing (); } From c282f77a6d444876c84f48305f630aff256639a5 Mon Sep 17 00:00:00 2001 From: "Mark A. Grondona" Date: Wed, 8 Jan 2025 15:18:53 -0800 Subject: [PATCH 4/8] librlist: add rlist_subtract() Problem: It would be useful to have an rlist diff routine the modifies the rlist argument, but this code is embedded in rlist_diff(), which returns a new rlist. Split rlist_subtract() out of rlist_diff() and make it public. Have rlist_diff() call rlist_subtract() internally. --- src/common/librlist/rlist.c | 24 ++++++++++++++---------- src/common/librlist/rlist.h | 5 +++++ 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/src/common/librlist/rlist.c b/src/common/librlist/rlist.c index 3232b6d00804..e46763b831bb 100644 --- a/src/common/librlist/rlist.c +++ b/src/common/librlist/rlist.c @@ -533,17 +533,9 @@ static struct rnode *rlist_detach_rank (struct rlist *rl, uint32_t rank) return n; } -struct rlist *rlist_diff (const struct rlist *rla, const struct rlist *rlb) +int rlist_subtract (struct rlist *rl, const struct rlist *rlb) { - struct rnode *n; - struct rlist *rl = rlist_create (); - - if (!rl || rlist_append (rl, rla) < 0) { - rlist_destroy (rl); - return NULL; - } - - n = zlistx_first (rlb->nodes); + struct rnode *n = zlistx_first (rlb->nodes); while (n) { /* Attempt to find and "detach" the rank which we're diffing. */ @@ -565,6 +557,18 @@ struct rlist *rlist_diff (const struct rlist *rla, const struct rlist *rlb) } n = zlistx_next (rlb->nodes); } + return 0; +} + +struct rlist *rlist_diff (const struct rlist *rla, const struct rlist *rlb) +{ + struct rlist *rl = rlist_create (); + if (!rl + || rlist_append (rl, rla) < 0 + || rlist_subtract (rl, rlb) < 0) { + rlist_destroy (rl); + return NULL; + } return rl; } diff --git a/src/common/librlist/rlist.h b/src/common/librlist/rlist.h index 9a1eb18c3901..d0f8a6212e59 100644 --- a/src/common/librlist/rlist.h +++ b/src/common/librlist/rlist.h @@ -151,6 +151,11 @@ int rlist_append (struct rlist *rl, const struct rlist *rl2); */ int rlist_add (struct rlist *rl, const struct rlist *rl2); +/* Subtract resources in `rl2` from `rl`. It is not an error if + * resources in rl2 are not present in `rl`. + */ +int rlist_subtract (struct rlist *rl, const struct rlist *rl2); + /* Return the set difference of 'rlb' from 'rla'. */ struct rlist *rlist_diff (const struct rlist *rla, const struct rlist *rlb); From 4e7dd92dcd751883c620ba76bf432cf63e58c699 Mon Sep 17 00:00:00 2001 From: "Mark A. Grondona" Date: Wed, 8 Jan 2025 15:08:58 -0800 Subject: [PATCH 5/8] resource: add subsystem for core reservation Problem: There is no way to reserve cores so they are not used for scheduling. Add a 'reserve' subsystem to the core resource module that takes a reserved set of cores via the 'cores[@ranks]' spec. Nothing is done with the reserve subsystem at this point. --- src/modules/resource/Makefile.am | 4 +- src/modules/resource/reserve.c | 81 ++++++++++++++++++++++++++++++++ src/modules/resource/reserve.h | 24 ++++++++++ 3 files changed, 108 insertions(+), 1 deletion(-) create mode 100644 src/modules/resource/reserve.c create mode 100644 src/modules/resource/reserve.h diff --git a/src/modules/resource/Makefile.am b/src/modules/resource/Makefile.am index 526a8e7a4cc6..fe944de245ee 100644 --- a/src/modules/resource/Makefile.am +++ b/src/modules/resource/Makefile.am @@ -40,7 +40,9 @@ libresource_la_SOURCES = \ drainset.h \ drainset.c \ upgrade.h \ - upgrade.c + upgrade.c \ + reserve.c \ + reserve.h TESTS = \ test_rutil.t \ diff --git a/src/modules/resource/reserve.c b/src/modules/resource/reserve.c new file mode 100644 index 000000000000..21918cb89d06 --- /dev/null +++ b/src/modules/resource/reserve.c @@ -0,0 +1,81 @@ +/************************************************************\ + * Copyright 2025 Lawrence Livermore National Security, LLC + * (c.f. AUTHORS, NOTICE.LLNS, COPYING) + * + * This file is part of the Flux resource manager framework. + * For details, see https://github.com/flux-framework. + * + * SPDX-License-Identifier: LGPL-3.0 +\************************************************************/ + +/* reserve.c - get static set of resources to reserve for the OS + */ + +#if HAVE_CONFIG_H +#include "config.h" +#endif +#include +#include + +#include "src/common/librlist/rlist.h" + +#include "resource.h" +#include "inventory.h" + +struct reserve { + struct resource_ctx *ctx; + struct rlist *rl; +}; + +const struct rlist *reserve_get (struct reserve *reserve) +{ + return reserve->rl; +} + +void reserve_destroy (struct reserve *reserve) +{ + if (reserve) { + int saved_errno = errno; + rlist_destroy (reserve->rl); + free (reserve); + errno = saved_errno; + } +} + +struct reserve *reserve_create (struct resource_ctx *ctx, + const char *spec) +{ + struct reserve *reserve; + struct rlist *rl = NULL; + + if (!(reserve = calloc (1, sizeof (*reserve)))) + return NULL; + reserve->ctx = ctx; + if (spec) { + flux_error_t error; + if (!(rl = rlist_from_json (inventory_get (ctx->inventory), NULL))) { + flux_log (ctx->h, + LOG_ERR, + "reserve: failed to get resources from inventory"); + goto error; + } + if (!(reserve->rl = rlist_copy_core_spec (rl, spec, &error))) { + flux_log (ctx->h, + LOG_ERR, + "error decoding reserve spec %s: %s", + spec, + error.text); + goto error; + } + rlist_destroy (rl); + } + return reserve; +error: + rlist_destroy (rl); + reserve_destroy (reserve); + return NULL; +} + +/* + * vi:tabstop=4 shiftwidth=4 expandtab + */ diff --git a/src/modules/resource/reserve.h b/src/modules/resource/reserve.h new file mode 100644 index 000000000000..6b001bdc44ea --- /dev/null +++ b/src/modules/resource/reserve.h @@ -0,0 +1,24 @@ +/************************************************************\ + * Copyright 2025 Lawrence Livermore National Security, LLC + * (c.f. AUTHORS, NOTICE.LLNS, COPYING) + * + * This file is part of the Flux resource manager framework. + * For details, see https://github.com/flux-framework. + * + * SPDX-License-Identifier: LGPL-3.0 +\************************************************************/ + +#ifndef _FLUX_RESOURCE_RESERVE_H +#define _FLUX_RESOURCE_RESERVE_H + +struct reserve *reserve_create (struct resource_ctx *ctx, const char *spec); +void reserve_destroy (struct reserve *exclude); + +const struct rlist *reserve_get (struct reserve *reserve); + +#endif /* !_FLUX_RESOURCE_RESERVE_H */ + +/* + * vi:tabstop=4 shiftwidth=4 expandtab + */ + From 3fa9bcebef29d3507fae323a2904697fae7c8b57 Mon Sep 17 00:00:00 2001 From: "Mark A. Grondona" Date: Wed, 8 Jan 2025 15:23:55 -0800 Subject: [PATCH 6/8] flux-resource: fix `rlist` named output format Problem: The `rlist` output format has `+` and `:` transposed, which results in an extraneous `+` in the output. Fix the format. --- src/cmd/flux-resource.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cmd/flux-resource.py b/src/cmd/flux-resource.py index 984f30e96d24..cb6417e8f36b 100755 --- a/src/cmd/flux-resource.py +++ b/src/cmd/flux-resource.py @@ -74,7 +74,7 @@ class FluxResourceConfig(UtilConfig): "description": "Format including resource list details", "format": ( "{state:>10} ?+:{queue:<5} ?:{propertiesx:<10.10+} {nnodes:>6} " - "+:{ncores:>6} ?:+{ngpus:>5} {rlist}" + "+:{ncores:>6} ?+:{ngpus:>5} {rlist}" ), }, } From b0b89e2ee90576a1588da47cc542179f9bf14348 Mon Sep 17 00:00:00 2001 From: "Mark A. Grondona" Date: Wed, 8 Jan 2025 15:29:00 -0800 Subject: [PATCH 7/8] resource: support resource.reserve configuration key Problem: There is no way to reserve a set of cores so they cannot be used by the scheduler of a Flux instance. Add support for a new config key `resource.reserve`, which takes a string of the form `cores[@ranks]` where `cores` and the optional `ranks` are idsets specifying the cores to reserve and the ranks on which to reserve them. If ranks is not specified, then the spec applies to all ranks. The reserved set of cores are subtracted from the resource set before it is handed off to the scheduler. The reserved resource set is also removed from the status response used by `flux resource list`. --- src/modules/resource/acquire.c | 9 +++++++-- src/modules/resource/resource.c | 14 +++++++++++++- src/modules/resource/resource.h | 2 ++ src/modules/resource/status.c | 12 ++++++++++-- 4 files changed, 32 insertions(+), 5 deletions(-) diff --git a/src/modules/resource/acquire.c b/src/modules/resource/acquire.c index a23261ada800..7b47432cb6a7 100644 --- a/src/modules/resource/acquire.c +++ b/src/modules/resource/acquire.c @@ -65,6 +65,7 @@ #include "reslog.h" #include "inventory.h" #include "exclude.h" +#include "reserve.h" #include "drain.h" #include "acquire.h" #include "monitor.h" @@ -108,6 +109,7 @@ static int acquire_request_init (struct acquire_request *ar, { struct resource_ctx *ctx = acquire->ctx; const struct idset *exclude = exclude_get (ctx->exclude); + const struct rlist *reserved = reserve_get (ctx->reserve); json_error_t e; struct rlist *rl; struct idset *drain = NULL; @@ -116,8 +118,11 @@ static int acquire_request_init (struct acquire_request *ar, errno = EINVAL; return -1; } - if (exclude && idset_count (exclude) > 0) { - (void)rlist_remove_ranks (rl, (struct idset *)exclude); + if (reserved || (exclude && idset_count (exclude) > 0)) { + if (exclude) + (void)rlist_remove_ranks (rl, (struct idset *)exclude); + if (reserved) + (void)rlist_subtract (rl, reserved); if (!(ar->resources = rlist_to_R (rl))) { errno = ENOMEM; goto error; diff --git a/src/modules/resource/resource.c b/src/modules/resource/resource.c index 8f3da8248307..05d9ce073310 100644 --- a/src/modules/resource/resource.c +++ b/src/modules/resource/resource.c @@ -36,12 +36,18 @@ #include "rutil.h" #include "status.h" #include "upgrade.h" +#include "reserve.h" /* Parse [resource] table. * * exclude = "targets" * Exclude specified broker rank(s) or hosts from scheduling * + * reserve = "corespec" + * Exclude specified coes from scheduling, corespec has the form: + * "cores[@ranks]", where each of cores and ranks are non-empty + * idsets. + * * [[resource.confg]] * Resource configuration array * @@ -71,6 +77,7 @@ static int parse_config (struct resource_ctx *ctx, const char *exclude = NULL; const char *path = NULL; const char *scheduling_path = NULL; + const char *reserve = NULL; int noverify = 0; int norestrict = 0; int no_update_watch = 0; @@ -80,12 +87,13 @@ static int parse_config (struct resource_ctx *ctx, if (flux_conf_unpack (conf, &error, - "{s?{s?s s?s s?o s?s s?b s?b s?b s?b !}}", + "{s?{s?s s?s s?o s?s s?s s?b s?b s?b s?b !}}", "resource", "path", &path, "scheduling", &scheduling_path, "config", &config, "exclude", &exclude, + "reserve", &reserve, "norestrict", &norestrict, "noverify", &noverify, "no-update-watch", &no_update_watch, @@ -147,6 +155,7 @@ static int parse_config (struct resource_ctx *ctx, } if (rconfig) { rconfig->exclude_idset = exclude; + rconfig->reserve = reserve; rconfig->noverify = noverify ? true : false; rconfig->norestrict = norestrict ? true : false; rconfig->no_update_watch = no_update_watch ? true : false; @@ -238,6 +247,7 @@ static void resource_ctx_destroy (struct resource_ctx *ctx) topo_destroy (ctx->topology); monitor_destroy (ctx->monitor); exclude_destroy (ctx->exclude); + reserve_destroy (ctx->reserve); reslog_destroy (ctx->reslog); inventory_destroy (ctx->inventory); flux_msg_handler_delvec (ctx->handlers); @@ -383,6 +393,8 @@ int mod_main (flux_t *h, int argc, char **argv) goto error; if (!(ctx->drain = drain_create (ctx, eventlog))) goto error; + if (!(ctx->reserve = reserve_create (ctx, config.reserve))) + goto error; } /* topology is initialized after exclude/drain etc since this * rank may attempt to drain itself due to a topology mismatch. diff --git a/src/modules/resource/resource.h b/src/modules/resource/resource.h index 41e627a209a6..33cee5bf0c5e 100644 --- a/src/modules/resource/resource.h +++ b/src/modules/resource/resource.h @@ -14,6 +14,7 @@ struct resource_config { json_t *R; const char *exclude_idset; + const char *reserve; bool rediscover; bool noverify; bool norestrict; @@ -29,6 +30,7 @@ struct resource_ctx { struct topo *topology; struct drain *drain; struct exclude *exclude; + struct reserve *reserve; struct acquire *acquire; struct reslog *reslog; struct status *status; diff --git a/src/modules/resource/status.c b/src/modules/resource/status.c index bdce54185f9e..053626748a21 100644 --- a/src/modules/resource/status.c +++ b/src/modules/resource/status.c @@ -21,6 +21,7 @@ #include "rutil.h" #include "monitor.h" #include "exclude.h" +#include "reserve.h" #include "status.h" #include "reslog.h" @@ -282,9 +283,11 @@ static json_t *update_properties_json (json_t *R, const struct rlist *all) /* Create an rlist object from R. Omit the scheduling key. Then: * - exclude the ranks in 'exclude' (if non-NULL) + * - exclude resources in 'reserved' (if non-NULL) */ static struct rlist *create_rlist (const json_t *R, - const struct idset *exclude) + const struct idset *exclude, + const struct rlist *reserved) { json_t *cpy; struct rlist *rl; @@ -302,6 +305,10 @@ static struct rlist *create_rlist (const json_t *R, if (rlist_remove_ranks (rl, (struct idset *)exclude) < 0) goto error; } + if (reserved) { + if (rlist_subtract (rl, reserved) < 0) + goto error; + } json_decref (cpy); return rl; error: @@ -316,9 +323,10 @@ static struct rlist *get_resource_list (struct status *status) if (!status->cache.rl) { const json_t *R; const struct idset *exclude = exclude_get (status->ctx->exclude); + const struct rlist *reserved = reserve_get (status->ctx->reserve); if ((R = inventory_get (status->ctx->inventory))) - status->cache.rl = create_rlist (R, exclude); + status->cache.rl = create_rlist (R, exclude, reserved); } return status->cache.rl; } From 6f88d307b46d04bb3e07eba2a2d8bb179eddb560 Mon Sep 17 00:00:00 2001 From: "Mark A. Grondona" Date: Fri, 10 Jan 2025 07:34:55 -0800 Subject: [PATCH 8/8] doc: document `resource.reserve` config key Problem: The `reserve` key in the `[resource]` table is not documented. Document it. --- doc/man5/flux-config-resource.rst | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/doc/man5/flux-config-resource.rst b/doc/man5/flux-config-resource.rst index 0f5e8ee9dbe0..a38473e57b45 100644 --- a/doc/man5/flux-config-resource.rst +++ b/doc/man5/flux-config-resource.rst @@ -79,6 +79,21 @@ rediscover (optional) If true, force rediscovery of resources using HWLOC, rather then using the R and HWLOC XML from the enclosing instance. +reserve + (optional) A string value that defines cores to reserve for the OS + and the broker ranks on which to reserve them. The argument is specified + as:: + + cores[@ranks] [cores[@ranks]]... + + where cores is an RFC 22 idset specifying the cores to reserve, and + the optional ranks is an RFC 22 idset specifying the ranks on which to + reserve them. If ``@ranks`` is not supplied, then cores will be reserved + on all ranks. Multiple instances of ``cores[@ranks]`` may be specified + (separated by whitespace) to reserve a different set of cores on different + ranks. For example, ``0-3@0 0`` would reserve cores 0-3 on rank 0 and + core 0 on all other ranks. + Note that updates to the resource table are ignored until the next Flux restart.