libflux: add flux_msg_is_local()

Problem: although the rexec service uses a private broker mechanism
to prevent remote use on rank 0, the sdbus service resides in a broker
module and doesn't have access to this mechanism and thus cannot
differentiate local vs remote clients.

Define a new RFC 12 role: FLUX_ROLE_LOCAL.  Set FLUX_ROLE_LOCAL in
message credentials when a message is received by the local connector
or sent by the broker or one of its modules.  Clear FLUX_ROLE_LOCAL
when a message is received by the broker overlay network.
Add an accessor flux_msg_is_local() that can be used to test whether
a message was sent locally or has transited brokers.

Fix two tests in t0017-security.t that expect a specific rolemask output
from ping, that now must include FLUX_ROLE_LOCAL.

Fixes #5136

Author: garlick

src/broker/broker.c

@@ -217,7 +217,7 @@ int main (int argc, char *argv[])
     ctx.cred.userid = getuid ();
     /* Set default rolemask for messages sent with flux_send()
      * on the broker's internal handle. */
-    ctx.cred.rolemask = FLUX_ROLE_OWNER;
+    ctx.cred.rolemask = FLUX_ROLE_OWNER | FLUX_ROLE_LOCAL;
 
     init_attrs (ctx.attrs, getpid (), &ctx.cred);
 

src/broker/module.c

@@ -658,7 +658,7 @@ module_t *module_add (modhash_t *mh,
      * credentials are always those of the instance owner.
      */
     p->cred.userid = getuid ();
-    p->cred.rolemask = FLUX_ROLE_OWNER;
+    p->cred.rolemask = FLUX_ROLE_OWNER | FLUX_ROLE_LOCAL;
 
     /* Update the modhash.
      */

src/broker/overlay.c

@@ -869,6 +869,18 @@ static void logdrop (struct overlay *ov,
               reason);
 }
 
+static int clear_msg_role (flux_msg_t *msg, uint32_t role)
+{
+    uint32_t rolemask;
+
+    if (flux_msg_get_rolemask (msg, &rolemask) < 0)
+        return -1;
+    rolemask &= ~role;
+    if (flux_msg_set_rolemask (msg, rolemask) < 0)
+        return -1;
+    return 0;
+}
+
 /* Handle a message received from TBON child (downstream).
  */
 static void child_cb (flux_reactor_t *r, flux_watcher_t *w,
@@ -883,6 +895,10 @@ static void child_cb (flux_reactor_t *r, flux_watcher_t *w,
 
     if (!(msg = zmqutil_msg_recv (ov->bind_zsock)))
         return;
+    if (clear_msg_role (msg, FLUX_ROLE_LOCAL) < 0) {
+        logdrop (ov, OVERLAY_DOWNSTREAM, msg, "failed to clear local role");
+        goto done;
+    }
     /* Flag this message as remotely received. This allows efficient
      * operation of the overlay_msg_is_local() function.
      */
@@ -992,6 +1008,10 @@ static void parent_cb (flux_reactor_t *r, flux_watcher_t *w,
 
     if (!(msg = zmqutil_msg_recv (ov->parent.zsock)))
         return;
+    if (clear_msg_role (msg, FLUX_ROLE_LOCAL) < 0) {
+        logdrop (ov, OVERLAY_UPSTREAM, msg, "failed to clear local role");
+        goto done;
+    }
     /* Flag this message as remotely received. This allows efficient
      * operation of the overlay_msg_is_local() function.
      */

src/common/libflux/message.c

@@ -507,6 +507,15 @@ int flux_msg_authorize (const flux_msg_t *msg, uint32_t userid)
     return 0;
 }
 
+bool flux_msg_is_local (const flux_msg_t *msg)
+{
+    uint32_t rolemask;
+    if (flux_msg_get_rolemask (msg, &rolemask) == 0
+        && (rolemask & FLUX_ROLE_LOCAL))
+        return true;
+    return false;
+}
+
 int flux_msg_set_nodeid (flux_msg_t *msg, uint32_t nodeid)
 {
     if (msg_validate (msg) < 0)

src/common/libflux/message.h

@@ -233,6 +233,7 @@ enum {
     FLUX_ROLE_NONE = 0,
     FLUX_ROLE_OWNER = 1,
     FLUX_ROLE_USER = 2,
+    FLUX_ROLE_LOCAL = 4,
     FLUX_ROLE_ALL = 0xFFFFFFFF,
 };
 int flux_msg_set_rolemask (flux_msg_t *msg, uint32_t rolemask);
@@ -260,6 +261,11 @@ int flux_msg_cred_authorize (struct flux_msg_cred cred, uint32_t userid);
  */
 int flux_msg_authorize (const flux_msg_t *msg, uint32_t userid);
 
+/* Return true if 'msg' credential carries FLUX_ROLE_LOCAL, indicating
+ * that the message has not traversed brokers.
+ */
+bool flux_msg_is_local (const flux_msg_t *msg);
+
 /* Get/set errnum (response only)
  */
 int flux_msg_set_errnum (flux_msg_t *msg, int errnum);

src/modules/connector-local/local.c

@@ -90,6 +90,11 @@ static int client_authenticate (struct connector_local *ctx,
         errno = EPERM;
         goto error;
     }
+    /* Tack on FLUX_ROLE_LOCAL to indicate that this message was
+     * accepted by the local connector.  This role is cleared when
+     * the message is received by another broker.
+     */
+    rolemask |= FLUX_ROLE_LOCAL;
     /* Test hook: drop owner cred for one connection.
      */
     if (flux_module_debug_test (ctx->h, DEBUG_OWNERDROP_ONESHOT, true)) {

t/t0017-security.t

@@ -95,12 +95,12 @@ test_expect_success 'simulated local connector auth failure returns EPERM' '
 
 test_expect_success 'flux ping --userid displays userid' '
 	flux ping --count=1 --userid broker >ping.out &&
-	grep -q "userid=$(id -u) rolemask=0x1" ping.out
+	grep -q "userid=$(id -u) rolemask=0x5" ping.out
 '
 
 test_expect_success 'FLUX_HANDLE_USERID can spoof userid in message' '
 	FLUX_HANDLE_USERID=9999 flux ping --count=1 --userid broker >ping2.out &&
-	grep -q "userid=9999 rolemask=0x1" ping2.out
+	grep -q "userid=9999 rolemask=0x5" ping2.out
 '
 
 test_expect_success 'FLUX_HANDLE_ROLEMASK can spoof rolemask in message' '