Skip to content

Commit 8cfd5b3

Browse files
garlickgarlick
committed
broker: add flub bootstrap method
Problem: there is no way to add brokers to an instance that has extra slots available. Add support for FLUB, the FLUx Bootstrap protocol, used when the broker is started with broker.boot-server=<uri> The bootstrap protocol consists of two RPCs: 1) overlay.flub-getinfo, which requests the allocation of an available rank from rank 0 of the instance that is being extended, and also retrieves the instance size and some broker attributes. 2) overlay.flub-kex, which exchanges public keys with the new rank's TBON parent and obtains the parent's TBON URI. Assumptions: - all ranks have the same topology configuration Limitations (for now): - only leaf nodes can be grafted with flub - hostnames will be logged as extra[0-N] - a broker rank cannot be re-allocated to a new broker - a broker cannot replace one that failed in a regular instance - dummy resources for the max size of the instance must be configured
1 parent bef904a commit 8cfd5b3

File tree

4 files changed

+200
-0
lines changed

4 files changed

+200
-0
lines changed

‎src/broker/Makefile.am‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,8 @@ libbroker_la_SOURCES = \
5656
boot_config.c \
5757
boot_pmi.h \
5858
boot_pmi.c \
59+
boot_flub.h \
60+
boot_flub.c \
5961
publisher.h \
6062
publisher.c \
6163
groups.h \

‎src/broker/boot_flub.c‎

Lines changed: 167 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,167 @@
1+
/************************************************************\
2+
* Copyright 2023 Lawrence Livermore National Security, LLC
3+
* (c.f. AUTHORS, NOTICE.LLNS, COPYING)
4+
*
5+
* This file is part of the Flux resource manager framework.
6+
* For details, see https://github.com/flux-framework.
7+
*
8+
* SPDX-License-Identifier: LGPL-3.0
9+
\************************************************************/
10+
11+
/* boot_flub.c - FLUx Boot protocol
12+
*
13+
* Add a broker to an existing Flux instance.
14+
*/
15+
16+
#if HAVE_CONFIG_H
17+
#include "config.h"
18+
#endif
19+
#include <errno.h>
20+
#include <jansson.h>
21+
#include <flux/core.h>
22+
23+
#include "src/common/libutil/errprintf.h"
24+
#include "src/common/libutil/errno_safe.h"
25+
#include "ccan/str/str.h"
26+
27+
#include "attr.h"
28+
#include "overlay.h"
29+
#include "topology.h"
30+
31+
#include "boot_flub.h"
32+
33+
struct boot_info {
34+
int size;
35+
int rank;
36+
json_t *attrs;
37+
};
38+
39+
struct boot_parent {
40+
char *pubkey;
41+
int rank;
42+
const char *uri;
43+
};
44+
45+
static int set_attrs (attr_t *attrs, json_t *dict)
46+
{
47+
const char *key;
48+
json_t *val;
49+
50+
json_object_foreach (dict, key, val) {
51+
const char *s = json_string_value (val);
52+
if (!s) {
53+
errno = EPROTO;
54+
return -1;
55+
}
56+
if (attr_add (attrs, key, s, ATTR_IMMUTABLE) < 0)
57+
return -1;
58+
}
59+
return 0;
60+
}
61+
62+
int boot_flub (struct broker *ctx, flux_error_t *error)
63+
{
64+
const char *uri = NULL;
65+
flux_t *h;
66+
flux_future_t *f = NULL;
67+
flux_future_t *f2 = NULL;
68+
struct topology *topo = NULL;
69+
const char *topo_uri;
70+
struct boot_info info;
71+
struct boot_parent parent;
72+
const char *bind_uri = NULL;
73+
int rc = -1;
74+
75+
/* Ask a Flux instance to allocate an available rank.
76+
*/
77+
(void)attr_get (ctx->attrs, "broker.boot-server", &uri, NULL);
78+
if (!uri)
79+
(void)attr_get (ctx->attrs, "parent-uri", &uri, NULL);
80+
if (!(h = flux_open_ex (uri, 0, error)))
81+
return -1;
82+
if (!(f = flux_rpc_pack (h,
83+
"overlay.flub-getinfo",
84+
0,
85+
0,
86+
"{}"))
87+
|| flux_rpc_get_unpack (f,
88+
"{s:i s:i s:o}",
89+
"rank", &info.rank,
90+
"size", &info.size,
91+
"attrs", &info.attrs) < 0) {
92+
errprintf (error, "%s", future_strerror (f, errno));
93+
goto out;
94+
}
95+
/* Set instance attributes obtained from boot server.
96+
*/
97+
if (set_attrs (ctx->attrs, info.attrs) < 0) {
98+
errprintf (error, "error setting attributes: %s", strerror (errno));
99+
goto out;
100+
}
101+
/* Create topology. All ranks are assumed to have the same topology.
102+
* The tbon.topo attribute is set in overlay_create() if not provided
103+
* on the command line.
104+
*/
105+
if (attr_get (ctx->attrs, "tbon.topo", &topo_uri, NULL) < 0) {
106+
errprintf (error, "error fetching tbon.topo attribute");
107+
goto out;
108+
}
109+
if (!(topo = topology_create (topo_uri, info.size, NULL))
110+
|| topology_set_rank (topo, info.rank) < 0
111+
|| overlay_set_topology (ctx->overlay, topo) < 0) {
112+
errprintf (error, "error creating topology: %s", strerror (errno));
113+
goto out;
114+
}
115+
if ((parent.rank = topology_get_parent (topo)) < 0) {
116+
errprintf (error,
117+
"rank %d has no parent in %s topology",
118+
info.rank,
119+
topo_uri);
120+
goto out;
121+
}
122+
/* Exchange public keys with TBON parent and obtain its URI.
123+
* FIXME: first wait for membership in broker.online group so
124+
* any start up order can be tolerated.
125+
*/
126+
if (!(f2 = flux_rpc_pack (h,
127+
"overlay.flub-kex",
128+
parent.rank,
129+
0,
130+
"{s:s s:s}",
131+
"name", overlay_cert_name (ctx->overlay),
132+
"pubkey", overlay_cert_pubkey (ctx->overlay)))
133+
|| flux_rpc_get_unpack (f2,
134+
"{s:s s:s}",
135+
"pubkey", &parent.pubkey,
136+
"uri", &parent.uri) < 0) {
137+
errprintf (error, "%s", future_strerror (f, errno));
138+
goto out;
139+
}
140+
/* Inform overlay subsystem of parent info.
141+
*/
142+
if (overlay_set_parent_uri (ctx->overlay, parent.uri) < 0
143+
|| overlay_set_parent_pubkey (ctx->overlay, parent.pubkey) < 0) {
144+
errprintf (error,
145+
"error setting up overlay parameters: %s",
146+
strerror (errno));
147+
goto out;
148+
}
149+
if (topology_get_child_ranks (topo, NULL, 0) > 0) {
150+
// FIXME: temporary restriction, avoiding bind complexity for now
151+
errprintf (error, "flub-booted brokers may not have children");
152+
goto out;
153+
}
154+
if (attr_add (ctx->attrs, "tbon.endpoint", bind_uri, ATTR_IMMUTABLE) < 0) {
155+
errprintf (error, "setattr tbon.endpoint");
156+
goto out;
157+
}
158+
rc = 0;
159+
out:
160+
topology_decref (topo);
161+
flux_future_destroy (f);
162+
flux_future_destroy (f2);
163+
flux_close (h);
164+
return rc;
165+
}
166+
167+
// vi:ts=4 sw=4 expandtab

‎src/broker/boot_flub.h‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
/************************************************************\
2+
* Copyright 2023 Lawrence Livermore National Security, LLC
3+
* (c.f. AUTHORS, NOTICE.LLNS, COPYING)
4+
*
5+
* This file is part of the Flux resource manager framework.
6+
* For details, see https://github.com/flux-framework.
7+
*
8+
* SPDX-License-Identifier: LGPL-3.0
9+
\************************************************************/
10+
11+
#ifndef BROKER_BOOT_FLUB_H
12+
#define BROKER_BOOT_FLUB_H
13+
14+
#include <flux/core.h>
15+
16+
#include "broker.h"
17+
18+
int boot_flub (struct broker *ctx, flux_error_t *error);
19+
20+
#endif /* BROKER_BOOT_FLUB_H */
21+
22+
// vi:ts=4 sw=4 expandtab

‎src/broker/broker.c‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@
5757
#include "runat.h"
5858
#include "heaptrace.h"
5959
#include "exec.h"
60+
#include "boot_flub.h"
6061
#include "boot_config.h"
6162
#include "boot_pmi.h"
6263
#include "publisher.h"
@@ -344,6 +345,8 @@ int main (int argc, char *argv[])
344345
if (attr_get (ctx.attrs, "broker.boot-method", &boot_method, NULL) < 0) {
345346
if (flux_conf_unpack (conf, NULL, "{s:{}}", "bootstrap") == 0)
346347
boot_method = "config";
348+
else if (attr_get (ctx.attrs, "broker.boot-server", NULL, NULL) == 0)
349+
boot_method = "flub";
347350
else
348351
boot_method = NULL;
349352
}
@@ -353,6 +356,12 @@ int main (int argc, char *argv[])
353356
goto cleanup;
354357
}
355358
}
359+
else if (boot_method && streq (boot_method, "flub")) {
360+
if (boot_flub (&ctx, &error) < 0) {
361+
log_msg ("boot-flub: %s", error.text);
362+
goto cleanup;
363+
}
364+
}
356365
else {
357366
if (boot_pmi (ctx.overlay, ctx.attrs) < 0) {
358367
log_msg ("boot-pmi failed");

0 commit comments

Comments
 (0)