broker: allow instance size > PMI bootstrap size

Problem: there is no way to bootstrap a flux instance using PMI
with ranks (initially) missing.

Allow the 'size' broker attribute to be set on the command line.
If set to a value greater than the PMI size, perform the PMI
exchange as usual with the PMI size, but configure the overlay
topology with the additional ranks.

Since 'hostlist' is an immutable attribute that is expected to
be set by the bootstrap implementation, set it to include placeholders
for the ranks that haven't connected yet "extra[0-N]" so we
get something other than "(null)" in the logs.

Author: garlick

src/broker/boot_pmi.c

@@ -75,17 +75,22 @@ static char *pmi_mapping_to_taskmap (const char *s)
 }
 
 /* Set broker.mapping attribute from enclosing instance taskmap.
+ * Skip setting the taskmap if extra_ranks is nonzero since the
+ * mapping of those ranks is unknown.  N.B. when broker.mapping is missing,
+ * use_ipc() below will return false, a good thing since it is unknown
+ * whether ipc:// would work for the TBON wire-up of extra nodes.
  */
 static int set_broker_mapping_attr (struct upmi *upmi,
                                     int size,
-                                    attr_t *attrs)
+                                    attr_t *attrs,
+                                    int extra_ranks)
 {
     char *val = NULL;
     int rc;
 
     if (size == 1)
         val = strdup ("[[0,1,1,1]]");
-    else {
+    else if (extra_ranks == 0) {
         /* First attempt to get flux.taskmap, falling back to
          * PMI_process_mapping if this key is not available.
          */
@@ -163,7 +168,7 @@ static int format_bind_uri (char *buf, int bufsz, attr_t *attrs, int rank)
     return -1;
 }
 
-static int set_hostlist_attr (attr_t *attrs, struct hostlist *hl)
+static int set_hostlist_attr (attr_t *attrs,struct hostlist *hl)
 {
     const char *value;
     char *s;
@@ -220,9 +225,10 @@ int boot_pmi (struct overlay *overlay, attr_t *attrs)
     int child_count;
     int *child_ranks = NULL;
     const char *uri;
-    int i;
     int upmi_flags = UPMI_LIBPMI_NOFLUX;
     const char *upmi_method;
+    const char *s;
+    int size;
 
     // N.B. overlay_create() sets the tbon.topo attribute
     if (attr_get (attrs, "tbon.topo", &topo_uri, NULL) < 0) {
@@ -249,11 +255,23 @@ int boot_pmi (struct overlay *overlay, attr_t *attrs)
         log_err ("set_instance_level_attr");
         goto error;
     }
-    if (set_broker_mapping_attr (upmi, info.size, attrs) < 0) {
+    /* Allow the PMI size to be overridden with a larger one so that
+     * additional ranks can be grafted on later.
+     */
+    size = info.size;
+    if (attr_get (attrs, "size", &s, NULL) == 0) {
+        errno = 0;
+        size = strtoul (s, NULL, 10);
+        if (errno != 0 || size <= info.size) {
+            log_msg ("instance size may only be increased");
+            goto error;
+        }
+    }
+    if (set_broker_mapping_attr (upmi, size, attrs, size - info.size) < 0) {
         log_err ("error setting broker.mapping attribute");
         goto error;
     }
-    if (!(topo = topology_create (topo_uri, info.size, &error))) {
+    if (!(topo = topology_create (topo_uri, size, &error))) {
         log_msg ("error creating '%s' topology: %s", topo_uri, error.text);
         goto error;
     }
@@ -269,16 +287,6 @@ int boot_pmi (struct overlay *overlay, attr_t *attrs)
         goto error;
     }
 
-    /* A size=1 instance has no peers, so skip the PMI exchange.
-     */
-    if (info.size == 1) {
-        if (hostlist_append (hl, hostname) < 0) {
-            log_err ("hostlist_append");
-            goto error;
-        }
-        goto done;
-    }
-
     /* Enable ipv6 for maximum flexibility in address selection.
      */
     overlay_set_ipv6 (overlay, 1);
@@ -310,6 +318,16 @@ int boot_pmi (struct overlay *overlay, attr_t *attrs)
         goto error;
     }
 
+    /* If the PMI size is 1, then skip the PMI exchange entirely.
+     */
+    if (info.size == 1) {
+        if (hostlist_append (hl, hostname) < 0) {
+            log_err ("hostlist_append");
+            goto error;
+        }
+        goto done;
+    }
+
     /* Each broker writes a "business card" consisting of hostname,
      * public key, and URI (empty string for leaf node).
      */
@@ -390,10 +408,13 @@ int boot_pmi (struct overlay *overlay, attr_t *attrs)
 
     /* Fetch the business card of children and inform overlay of public keys.
      */
-    for (i = 0; i < child_count; i++) {
+    for (int i = 0; i < child_count; i++) {
         const char *peer_pubkey;
         int child_rank = child_ranks[i];
 
+        if (child_rank >= info.size)
+            break;
+
         if (snprintf (key, sizeof (key), "%d", child_rank) >= sizeof (key)) {
             log_msg ("pmi key string overflow");
             goto error;
@@ -428,7 +449,7 @@ int boot_pmi (struct overlay *overlay, attr_t *attrs)
     /* Fetch the business card of all ranks and build hostlist.
      * The hostlist is built independently (and in parallel) on all ranks.
      */
-    for (i = 0; i < info.size; i++) {
+    for (int i = 0; i < info.size; i++) {
         const char *peer_hostname;
 
         if (snprintf (key, sizeof (key), "%d", i) >= sizeof (key)) {
@@ -471,6 +492,17 @@ int boot_pmi (struct overlay *overlay, attr_t *attrs)
     }
 
 done:
+    /* If the instance size is greater than the PMI size, add placeholder
+     * names to the hostlist for the ranks that haven't joined yet.
+     */
+    for (int i = info.size; i < size; i++) {
+        char buf[64];
+        snprintf (buf, sizeof (buf), "extra%d", i - info.size);
+        if (hostlist_append (hl, buf) < 0) {
+            log_err ("hostlist_append");
+            goto error;
+        }
+    }
     if (set_hostlist_attr (attrs, hl) < 0) {
         log_err ("setattr hostlist");
         goto error;

src/broker/overlay.c

@@ -1590,6 +1590,7 @@ int overlay_register_attrs (struct overlay *overlay)
                          overlay->rank,
                          ATTR_IMMUTABLE) < 0)
         return -1;
+    (void)attr_delete (overlay->attrs, "size", true);
     if (attr_add_uint32 (overlay->attrs,
                          "size", overlay->size,
                          ATTR_IMMUTABLE) < 0)