API+linux: add new THISSYSTEM_ALLOWED_RESOURCES flag

XML is useful for improving discovery time, but it doesn't work
when the XML should be adapted to different cgroups for different
jobs on the same machine.

This new flag (combined with IS_THISSYSTEM) (or the corresponding env var)
load normal XML or synthetic and apply cpu/cgroup restrictions that the
actual machine reports on Linux.

Note that the XML must be created without any restriction (outside of any
cgroup that disallows some PUs or NUMAs on Linux).

Signed-off-by: Brice Goglin <[email protected]>

Author: bgoglin

doc/Makefile.am

@@ -1,4 +1,4 @@
-# Copyright © 2009-2016 Inria.  All rights reserved.
+# Copyright © 2009-2017 Inria.  All rights reserved.
 # Copyright © 2009-2013 Université Bordeaux
 # Copyright © 2009-2016 Cisco Systems, Inc.  All rights reserved.
 # See COPYING in top-level directory.
@@ -430,8 +430,9 @@ man3_setsource_DATA = \
 man3_configurationdir = $(man3dir)
 man3_configuration_DATA = \
         $(DOX_MAN_DIR)/man3/hwlocality_configuration.3 \
-        $(DOX_MAN_DIR)/man3/HWLOC_TOPOLOGY_FLAG_IS_THISSYSTEM.3 \
         $(DOX_MAN_DIR)/man3/HWLOC_TOPOLOGY_FLAG_WHOLE_SYSTEM.3 \
+        $(DOX_MAN_DIR)/man3/HWLOC_TOPOLOGY_FLAG_IS_THISSYSTEM.3 \
+        $(DOX_MAN_DIR)/man3/HWLOC_TOPOLOGY_FLAG_THISSYSTEM_ALLOWED_RESOURCES.3 \
         $(DOX_MAN_DIR)/man3/hwloc_topology_flags_e.3 \
         $(DOX_MAN_DIR)/man3/hwloc_topology_get_flags.3 \
         $(DOX_MAN_DIR)/man3/hwloc_topology_set_flags.3 \

doc/hwloc.doxy

@@ -844,6 +844,20 @@ following environment variables.
   This can be used for efficiency reasons to first detect the topology once,
   save it to an XML file, and quickly reload it later through the XML
   backend, but still having binding functions actually do bind.
+  This also enables support for the variable HWLOC_THISSYSTEM_ALLOWED_RESOURCES.
+  </dd>
+
+<dt>HWLOC_THISSYSTEM_ALLOWED_RESOURCES=1</dt>
+  <dd>Get the set of allowed resources from the native operating system
+  even if the topology was loaded from XML or synthetic description,
+  as if ::HWLOC_TOPOLOGY_FLAG_THISSYSTEM_ALLOWED_RESOURCES was set
+  with hwloc_topology_set_flags().
+  This variable requires the topology to match the current system
+  (see the variable HWLOC_THISSYSTEM).
+  This is useful when the topology is not loaded directly from the
+  local machine (e.g. for performance reason) and it comes with all
+  resources, but the running process is restricted to only a part
+  of the machine (for instance because of Linux Cgroup/Cpuset).
   </dd>
 
 <dt>HWLOC_HIDE_ERRORS=0</dt>
@@ -1662,6 +1676,12 @@ This behavior may be reverted by asserting that loaded file really
 matches the underlying system with the HWLOC_THISSYSTEM environment
 variable or the ::HWLOC_TOPOLOGY_FLAG_IS_THISSYSTEM topology flag.
 
+\note The topology flag ::HWLOC_TOPOLOGY_FLAG_THISSYSTEM_ALLOWED_RESOURCES
+may be used to load a XML topology that contains the entire machine
+and restrict it to the part that is actually available to the current
+process (e.g. when Linux Cgroup/Cpuset are used to restrict the set
+of resources).
+
 \note hwloc also offers the ability to export/import \ref hwlocality_diff.
 
 \note XML topology files are not localized. They use a dot as a
@@ -2819,6 +2839,12 @@ interface, and the import/export may also be directed to memory buffer
 (that may for instance be transmitted between applications through a package).
 See also \ref xml.
 
+\note The environment variable HWLOC_THISSYSTEM_ALLOWED_RESOURCES
+may be used to load a XML topology that contains the entire machine
+and restrict it to the part that is actually available to the current
+process (e.g. when Linux Cgroup/Cpuset are used to restrict the set
+of resources). See \ref envvar.
+
 
 \subsection faq_multitopo How many topologies may I use in my program?
 

hwloc/topology-linux.c

@@ -1956,6 +1956,49 @@ hwloc_linux_get_area_memlocation(hwloc_topology_t topology __hwloc_attribute_unu
   return ret;
 }
 
+static void hwloc_linux__get_allowed_resources(hwloc_topology_t topology, const char *root_path, int root_fd, char **cpuset_namep);
+
+static int hwloc_linux_get_allowed_resources_hook(hwloc_topology_t topology)
+{
+  const char *fsroot_path;
+  char *cpuset_name;
+  int root_fd = -1;
+
+  fsroot_path = getenv("HWLOC_FSROOT");
+  if (!fsroot_path)
+    fsroot_path = "/";
+
+#ifdef HAVE_OPENAT
+  root_fd = open(fsroot_path, O_RDONLY | O_DIRECTORY);
+  if (root_fd < 0)
+    goto out;
+#else
+  if (strcmp(fsroot_path, "/")) {
+    errno = ENOSYS;
+    goto out;
+  }
+#endif
+
+  /* we could also error-out if the current topology doesn't actually match the system,
+   * at least for PUs and NUMA nodes. But it would increase the overhead of loading XMLs.
+   *
+   * Just trust the user when he sets THISSYSTEM=1. It enables hacky
+   * tests such as restricting random XML or synthetic to the current
+   * machine (uses the default cgroup).
+   */
+
+  hwloc_linux__get_allowed_resources(topology, fsroot_path, root_fd, &cpuset_name);
+  if (cpuset_name) {
+    hwloc_obj_add_info(topology->levels[0][0], "LinuxCgroup", cpuset_name);
+    free(cpuset_name);
+  }
+  if (root_fd != -1)
+    close(root_fd);
+
+ out:
+  return -1;
+}
+
 void
 hwloc_set_linuxfs_hooks(struct hwloc_binding_hooks *hooks,
 			struct hwloc_topology_support *support __hwloc_attribute_unused)
@@ -1987,6 +2030,7 @@ hwloc_set_linuxfs_hooks(struct hwloc_binding_hooks *hooks,
   support->membind->bind_membind = 1;
   support->membind->interleave_membind = 1;
   support->membind->migrate_membind = 1;
+  hooks->get_allowed_resources = hwloc_linux_get_allowed_resources_hook;
 }
 
 
@@ -2152,7 +2196,7 @@ hwloc_read_linux_cpuset_name(int fsroot_fd, hwloc_pid_t pid)
  * are cgroup<name>/cpuset.{cpus,mems} or cpuset<name>/{cpus,mems} files.
  */
 static void
-hwloc_admin_disable_set_from_cpuset(struct hwloc_linux_backend_data_s *data,
+hwloc_admin_disable_set_from_cpuset(int root_fd,
 				    const char *cgroup_mntpnt, const char *cpuset_mntpnt, const char *cpuset_name,
 				    const char *attr_name,
 				    hwloc_bitmap_t admin_enabled_cpus_set)
@@ -2172,7 +2216,7 @@ hwloc_admin_disable_set_from_cpuset(struct hwloc_linux_backend_data_s *data,
     hwloc_debug("Trying to read cpuset file <%s>\n", cpuset_filename);
   }
 
-  fd = hwloc_open(cpuset_filename, data->root_fd);
+  fd = hwloc_open(cpuset_filename, root_fd);
   if (fd < 0) {
     /* found no cpuset description, ignore it */
     hwloc_debug("Couldn't find cpuset <%s> description, ignoring\n", cpuset_name);
@@ -4152,13 +4196,29 @@ hwloc_linux_try_hardwired_cpuinfo(struct hwloc_backend *backend)
   return -1;
 }
 
+static void hwloc_linux__get_allowed_resources(hwloc_topology_t topology, const char *root_path, int root_fd, char **cpuset_namep)
+{
+  char *cpuset_mntpnt, *cgroup_mntpnt, *cpuset_name = NULL;
+  hwloc_find_linux_cpuset_mntpnt(&cgroup_mntpnt, &cpuset_mntpnt, root_path);
+  if (cgroup_mntpnt || cpuset_mntpnt) {
+    cpuset_name = hwloc_read_linux_cpuset_name(root_fd, topology->pid);
+    if (cpuset_name) {
+      hwloc_admin_disable_set_from_cpuset(root_fd, cgroup_mntpnt, cpuset_mntpnt, cpuset_name, "cpus", topology->levels[0][0]->allowed_cpuset);
+      hwloc_admin_disable_set_from_cpuset(root_fd, cgroup_mntpnt, cpuset_mntpnt, cpuset_name, "mems", topology->levels[0][0]->allowed_nodeset);
+    }
+    free(cgroup_mntpnt);
+    free(cpuset_mntpnt);
+  }
+  *cpuset_namep = cpuset_name;
+}
+
 static int
 hwloc_look_linuxfs(struct hwloc_backend *backend)
 {
   struct hwloc_topology *topology = backend->topology;
   struct hwloc_linux_backend_data_s *data = backend->private_data;
   unsigned nbnodes;
-  char *cpuset_mntpnt, *cgroup_mntpnt, *cpuset_name = NULL;
+  char *cpuset_name;
   struct hwloc_linux_cpuinfo_proc * Lprocs = NULL;
   struct hwloc_obj_info_s *global_infos = NULL;
   unsigned global_infos_count = 0;
@@ -4226,16 +4286,7 @@ hwloc_look_linuxfs(struct hwloc_backend *backend)
   /**********************
    * Gather the list of admin-disabled cpus and mems
    */
-  hwloc_find_linux_cpuset_mntpnt(&cgroup_mntpnt, &cpuset_mntpnt, data->root_path);
-  if (cgroup_mntpnt || cpuset_mntpnt) {
-    cpuset_name = hwloc_read_linux_cpuset_name(data->root_fd, topology->pid);
-    if (cpuset_name) {
-      hwloc_admin_disable_set_from_cpuset(data, cgroup_mntpnt, cpuset_mntpnt, cpuset_name, "cpus", topology->levels[0][0]->allowed_cpuset);
-      hwloc_admin_disable_set_from_cpuset(data, cgroup_mntpnt, cpuset_mntpnt, cpuset_name, "mems", topology->levels[0][0]->allowed_nodeset);
-    }
-    free(cgroup_mntpnt);
-    free(cpuset_mntpnt);
-  }
+  hwloc_linux__get_allowed_resources(topology, data->root_path, data->root_fd, &cpuset_name);
 
   /*********************
    * Memory information

hwloc/topology.c

@@ -1,6 +1,6 @@
 /*
  * Copyright © 2009 CNRS
- * Copyright © 2009-2016 Inria.  All rights reserved.
+ * Copyright © 2009-2017 Inria.  All rights reserved.
  * Copyright © 2009-2012 Université Bordeaux
  * Copyright © 2009-2011 Cisco Systems, Inc.  All rights reserved.
  * See COPYING in top-level directory.
@@ -2618,6 +2618,12 @@ hwloc_discover(struct hwloc_topology *topology)
     return -1;
   }
 
+  if (topology->binding_hooks.get_allowed_resources && topology->is_thissystem) {
+    const char *env = getenv("HWLOC_THISSYSTEM_ALLOWED_RESOURCES");
+    if ((env && atoi(env))
+	|| (topology->flags & HWLOC_TOPOLOGY_FLAG_THISSYSTEM_ALLOWED_RESOURCES))
+      topology->binding_hooks.get_allowed_resources(topology);
+  }
   hwloc_debug("%s", "\nPropagate disallowed cpus down and up\n");
   hwloc_bitmap_and(topology->levels[0][0]->allowed_cpuset, topology->levels[0][0]->allowed_cpuset, topology->levels[0][0]->cpuset);
   propagate_unused_cpuset(topology->levels[0][0], NULL);
@@ -2905,7 +2911,7 @@ hwloc_topology_set_flags (struct hwloc_topology *topology, unsigned long flags)
     return -1;
   }
 
-  if (flags & ~(HWLOC_TOPOLOGY_FLAG_WHOLE_SYSTEM|HWLOC_TOPOLOGY_FLAG_IS_THISSYSTEM)) {
+  if (flags & ~(HWLOC_TOPOLOGY_FLAG_WHOLE_SYSTEM|HWLOC_TOPOLOGY_FLAG_IS_THISSYSTEM|HWLOC_TOPOLOGY_FLAG_THISSYSTEM_ALLOWED_RESOURCES)) {
     errno = EINVAL;
     return -1;
   }

include/hwloc.h

@@ -1738,7 +1738,28 @@ enum hwloc_topology_flags_e {
    * backend, but still having binding functions actually do bind.
    * \hideinitializer
    */
-  HWLOC_TOPOLOGY_FLAG_IS_THISSYSTEM = (1UL<<1)
+  HWLOC_TOPOLOGY_FLAG_IS_THISSYSTEM = (1UL<<1),
+
+ /** \brief Get the set of allowed resources from the local operating system even if the topology was loaded from XML or synthetic description.
+   *
+   * If the topology was loaded from XML or from a synthetic string,
+   * restrict it by applying the current process restrictions such as
+   * Linux Cgroup/Cpuset.
+   *
+   * This is useful when the topology is not loaded directly from
+   * the local machine (e.g. for performance reason) and it comes
+   * with all resources, while the running process is restricted
+   * to only parts of the machine.
+   *
+   * This flag is ignored unless ::HWLOC_TOPOLOGY_FLAG_IS_THISSYSTEM is
+   * also set since the loaded topology must match the underlying machine
+   * where restrictions will be gathered from.
+   *
+   * Setting the environment variable HWLOC_THISSYSTEM_ALLOWED_RESOURCES
+   * would result in the same behavior.
+   * \hideinitializer
+   */
+  HWLOC_TOPOLOGY_FLAG_THISSYSTEM_ALLOWED_RESOURCES = (1UL<<2)
 };
 
 /** \brief Set OR'ed flags to non-yet-loaded topology.

include/hwloc/rename.h

@@ -1,6 +1,6 @@
 /*
  * Copyright © 2009-2011 Cisco Systems, Inc.  All rights reserved.
- * Copyright © 2010-2016 Inria.  All rights reserved.
+ * Copyright © 2010-2017 Inria.  All rights reserved.
  * See COPYING in top-level directory.
  */
 
@@ -119,6 +119,7 @@ extern "C" {
 
 #define HWLOC_TOPOLOGY_FLAG_WHOLE_SYSTEM HWLOC_NAME_CAPS(TOPOLOGY_FLAG_WHOLE_SYSTEM)
 #define HWLOC_TOPOLOGY_FLAG_IS_THISSYSTEM HWLOC_NAME_CAPS(TOPOLOGY_FLAG_IS_THISSYSTEM)
+#define HWLOC_TOPOLOGY_FLAG_THISSYSTEM_ALLOWED_RESOURCES HWLOC_NAME_CAPS(TOPOLOGY_FLAG_THISSYSTEM_ALLOWED_RESOURCES)
 
 #define hwloc_topology_set_pid HWLOC_NAME(topology_set_pid)
 #define hwloc_topology_set_synthetic HWLOC_NAME(topology_set_synthetic)

include/private/private.h

@@ -1,6 +1,6 @@
 /*
  * Copyright © 2009      CNRS
- * Copyright © 2009-2016 Inria.  All rights reserved.
+ * Copyright © 2009-2017 Inria.  All rights reserved.
  * Copyright © 2009-2012 Université Bordeaux
  * Copyright © 2009-2011 Cisco Systems, Inc.  All rights reserved.
  *
@@ -110,6 +110,8 @@ struct hwloc_topology {
      * see hwloc_alloc_or_fail which is convenient for that.  */
     void *(*alloc_membind)(hwloc_topology_t topology, size_t len, hwloc_const_nodeset_t nodeset, hwloc_membind_policy_t policy, int flags);
     int (*free_membind)(hwloc_topology_t topology, void *addr, size_t len);
+
+    int (*get_allowed_resources)(hwloc_topology_t topology);
   } binding_hooks;
 
   struct hwloc_topology_support support;