Merge pull request #4538 from grondo/issue#4536

resource: add norestrict option to avoid restricting hwloc topology XML

Author: mergify[bot]

doc/man5/flux-config-resource.rst

@@ -32,6 +32,12 @@ exclude
    the scheduler, but will still be used to determine satisfiability of job
    requests until the instance is restarted.
 
+norestrict
+   (optional) Disable restricting of the loaded HWLOC topology XML to the
+   current cpu affinity mask of the Flux broker. This option should be used
+   when the Flux system instance is constrained to a subset of cores,
+   but jobs run within this instance should have access to all cores.
+
 
 EXAMPLE
 =======
@@ -41,6 +47,7 @@ EXAMPLE
    [resource]
    path = "/etc/flux/system/R"
    exclude = "test[3,108]"
+   norestrict = true
 
 
 RESOURCES

doc/test/spell.en.pws

@@ -634,3 +634,4 @@ pbatch
 pdebug
 parentof
 bg
+norestrict

src/cmd/flux-R.c

@@ -364,7 +364,7 @@ static char *get_xml (optparse_t *p)
             log_err_exit ("failed to read XML");
     }
     else if (optparse_hasopt (p, "local")) {
-        if (!(s = rhwloc_local_topology_xml ()))
+        if (!(s = rhwloc_local_topology_xml (0)))
             log_err_exit ("failed to gather local topology XML");
     }
 

src/common/librlist/rhwloc.c

@@ -14,7 +14,7 @@
 
 #include <flux/idset.h>
 
-#include "hwloc.h"
+#include "rhwloc.h"
 
 /*  Common hwloc_topology_init() and flags for Flux hwloc usage:
  */
@@ -65,7 +65,8 @@ hwloc_topology_t rhwloc_xml_topology_load (const char *xml)
     return topo;
 }
 
-hwloc_topology_t rhwloc_local_topology_load (void)
+
+hwloc_topology_t rhwloc_local_topology_load (rhwloc_flags_t flags)
 {
     hwloc_topology_t topo = NULL;
     hwloc_bitmap_t rset = NULL;
@@ -87,6 +88,8 @@ hwloc_topology_t rhwloc_local_topology_load (void)
 #endif
     if (hwloc_topology_load (topo) < 0)
         goto err;
+    if (flags & RHWLOC_NO_RESTRICT)
+        return (topo);
     if (!(rset = hwloc_bitmap_alloc ())
         || (hwloc_get_cpubind (topo, rset, HWLOC_CPUBIND_PROCESS) < 0))
         goto err;
@@ -100,12 +103,12 @@ hwloc_topology_t rhwloc_local_topology_load (void)
     return NULL;
 }
 
-char *rhwloc_local_topology_xml (void)
+char *rhwloc_local_topology_xml (rhwloc_flags_t rflags)
 {
     char *buf;
     int buflen;
     char *copy;
-    hwloc_topology_t topo = rhwloc_local_topology_load ();
+    hwloc_topology_t topo = rhwloc_local_topology_load (rflags);
     if (topo == NULL)
         return (NULL);
 #if HWLOC_API_VERSION >= 0x20000

src/common/librlist/rhwloc.h

@@ -13,17 +13,21 @@
 
 #include <hwloc.h>
 
+typedef enum {
+    RHWLOC_NO_RESTRICT = 0x1
+} rhwloc_flags_t;
+
 /*  Load local topology with Flux standard flags and filtering
  */
-hwloc_topology_t rhwloc_local_topology_load (void);
+hwloc_topology_t rhwloc_local_topology_load (rhwloc_flags_t flags);
 
 /*  As above, but return hwloc_topoology_t from XML
  */
 hwloc_topology_t rhwloc_xml_topology_load (const char *xml);
 
 /*  Load local topology and return XML as allocated string
  */
-char *rhwloc_local_topology_xml (void);
+char *rhwloc_local_topology_xml (rhwloc_flags_t flags);
 
 /*  Return HostName from an hwloc topology object
  */

src/common/librlist/rlist.c

@@ -2269,7 +2269,7 @@ struct rlist *rlist_from_hwloc (int rank, const char *xml)
     if (xml)
         topo = rhwloc_xml_topology_load (xml);
     else
-        topo = rhwloc_local_topology_load ();
+        topo = rhwloc_local_topology_load (0);
     if (!topo)
         goto fail;
     if (!(ids = rhwloc_core_idset_string (topo))

src/common/librlist/test/rhwloc.c

@@ -781,7 +781,7 @@ void test_xml ()
     struct rlist *rl2 = NULL;
     char *s1 = NULL;
     char *s2 = NULL;
-    char *xml = rhwloc_local_topology_xml ();
+    char *xml = rhwloc_local_topology_xml (0);
     if (!xml)
         BAIL_OUT ("rhwloc_local_topology_xml failed!");
     pass ("rhwloc_topology_xml");

src/modules/resource/resource.c

@@ -43,26 +43,32 @@
  *
  * noverify = true
  *   Skip verification that configured resources match local hwloc
+ *
+ * norestrict = false
+ *   When generating hwloc topology XML, do not restrict to current cpumask
  */
 static int parse_config (struct resource_ctx *ctx,
                          const flux_conf_t *conf,
                          const char **excludep,
                          json_t **R,
                          bool *noverifyp,
+                         bool *norestrictp,
                          flux_error_t *errp)
 {
     flux_error_t error;
     const char *exclude  = NULL;
     const char *path = NULL;
     int noverify = 0;
+    int norestrict = 0;
     json_t *o = NULL;
 
     if (flux_conf_unpack (conf,
                           &error,
-                          "{s?:{s?:s s?:s s?:b !}}",
+                          "{s?:{s?:s s?:s s?:b s?b !}}",
                           "resource",
                             "path", &path,
                             "exclude", &exclude,
+                            "norestrict", &norestrict,
                             "noverify", &noverify) < 0) {
         errprintf (errp,
                    "error parsing [resource] configuration: %s",
@@ -97,6 +103,8 @@ static int parse_config (struct resource_ctx *ctx,
     *excludep = exclude;
     if (noverifyp)
         *noverifyp = noverify ? true : false;
+    if (norestrictp)
+        *norestrictp = norestrict ? true : false;
     if (R)
         *R = o;
     return 0;
@@ -126,6 +134,7 @@ static void config_reload_cb (flux_t *h,
                       &exclude,
                       NULL,
                       NULL,
+                      NULL,
                       &error) < 0) {
         errstr = error.text;
         goto error;
@@ -397,6 +406,7 @@ int mod_main (flux_t *h, int argc, char **argv)
     json_t *eventlog = NULL;
     bool monitor_force_up = false;
     bool noverify = false;
+    bool norestrict = false;
     json_t *R_from_config;
 
     if (!(ctx = resource_ctx_create (h)))
@@ -410,6 +420,7 @@ int mod_main (flux_t *h, int argc, char **argv)
                       &exclude_idset,
                       &R_from_config,
                       &noverify,
+                      &norestrict,
                       &error) < 0) {
         flux_log (h, LOG_ERR, "%s", error.text);
         goto error;
@@ -426,7 +437,7 @@ int mod_main (flux_t *h, int argc, char **argv)
     }
     if (!(ctx->inventory = inventory_create (ctx, R_from_config)))
         goto error;
-    if (!(ctx->topology = topo_create (ctx, noverify)))
+    if (!(ctx->topology = topo_create (ctx, noverify, norestrict)))
         goto error;
     if (!(ctx->monitor = monitor_create (ctx, monitor_force_up)))
         goto error;

src/modules/resource/topo.c

@@ -22,7 +22,6 @@
 #endif
 #include <jansson.h>
 #include <flux/core.h>
-#include <hwloc.h>
 
 #include "src/common/libidset/idset.h"
 #include "src/common/libutil/errno_safe.h"
@@ -275,15 +274,18 @@ void topo_destroy (struct topo *topo)
     }
 }
 
-struct topo *topo_create (struct resource_ctx *ctx, bool no_verify)
+struct topo *topo_create (struct resource_ctx *ctx,
+                          bool no_verify,
+                          bool no_restrict)
 {
     struct topo *topo;
     json_t *R;
+    int flags = no_restrict ? RHWLOC_NO_RESTRICT : 0;
 
     if (!(topo = calloc (1, sizeof (*topo))))
         return NULL;
     topo->ctx = ctx;
-    if (!(topo->xml = rhwloc_local_topology_xml ())) {
+    if (!(topo->xml = rhwloc_local_topology_xml (flags))) {
         flux_log_error (ctx->h, "error loading hwloc topology");
         goto error;
     }

src/modules/resource/topo.h

@@ -11,7 +11,9 @@
 #ifndef _FLUX_RESOURCE_TOPO_H
 #define _FLUX_RESOURCE_TOPO_H
 
-struct topo *topo_create (struct resource_ctx *ctx, bool no_verify);
+struct topo *topo_create (struct resource_ctx *ctx,
+                          bool no_verify,
+                          bool no_restrict);
 void topo_destroy (struct topo *topo);