Merge pull request #5253 from grondo/issue#5250

support utf-8 in broker logs

Author: mergify[bot]

src/broker/broker.c

@@ -13,6 +13,7 @@
 #endif
 #include <assert.h>
 #include <libgen.h>
+#include <locale.h>
 #include <inttypes.h>
 #include <sys/prctl.h>
 #include <sys/resource.h>
@@ -204,6 +205,8 @@ int main (int argc, char *argv[])
     const char *method;
     flux_error_t error;
 
+    setlocale (LC_ALL, "");
+
     memset (&ctx, 0, sizeof (ctx));
     log_init (argv[0]);
 

src/common/libutil/stdlog.c

@@ -180,8 +180,6 @@ int stdlog_vencodef (char *buf, int len, struct stdlog_header *hdr,
     rc += n;
     if (n > len - m)
         n = len - m;
-    for (i = 0; i < n; i++)
-        buf[m + i] &= 0x7f; // ensure only ascii chars are logged
     for (i = n - 1; i >= 0; i--) {
         if (buf[m + i] != '\r' && buf[m + i] != '\n')
             break;

src/common/libutil/test/stdlog.c

@@ -127,28 +127,57 @@ int main(int argc, char** argv)
         "stdlog_decode decoded procid") ;
     ok (streq (hdr.msgid, cln.msgid),
         "stdlog_decode decoded msgid") ;
-    ok (sdlen == strlen (STDLOG_NILVALUE) && strncmp (sd, STDLOG_NILVALUE, sdlen) == 0,
+    ok (sdlen == strlen (STDLOG_NILVALUE)
+        && strncmp (sd, STDLOG_NILVALUE, sdlen) == 0,
         "stdlog_decode decoded structured data");
-    ok (msglen == strlen (STDLOG_NILVALUE) && strncmp (msg, STDLOG_NILVALUE, msglen) == 0,
+    ok (msglen == strlen (STDLOG_NILVALUE)
+        && strncmp (msg, STDLOG_NILVALUE, msglen) == 0,
         "stdlog_decode decoded message");
 
     /* Check that trailing \n or \r in message are dropped
      */
     stdlog_init (&hdr);
-    len = stdlog_encode (buf, sizeof (buf), &hdr,
+    len = stdlog_encode (buf,
+                         sizeof (buf),
+                         &hdr,
                          STDLOG_NILVALUE,
                          "Hello whorl\n\r\n");
     ok (len >= 0,
         "stdlog_encode worked with message");
     diag ("%.*s", len, buf);
     n = stdlog_decode (buf, len, &hdr, &sd, &sdlen, &msg, &msglen);
-    ok (n == 0 && strncmp (msg, "Hello whorl", msglen) == 0,
+    ok (n == 0
+        && strncmp (msg, "Hello whorl", msglen) == 0,
         "trailing cr/lf chars were truncated");
 
+    /* Check that valid UTF-8 non-ascii characters are preserved
+     */
+    const char data[] = "jobid ƒ6LEmNENaf9 😄 😹";
+    len = stdlog_encode (buf,
+                         sizeof (buf),
+                         &hdr,
+                         STDLOG_NILVALUE,
+                         data);
+    ok (len >= 0,
+        "stdlog_encode worked with %s", data);
+    n = stdlog_decode (buf, len, &hdr, &sd, &sdlen, &msg, &msglen);
+    ok (n == 0,
+        "stdlog_decode worked");
+    is (data, msg,
+        "non-ascii characters were preserved");
+
     int i = 0;
     while (valid[i] != NULL) {
-        n = stdlog_decode (valid[i], strlen (valid[i]), &hdr, &sd, &sdlen, &msg, &msglen);
-        ok (n == 0 && msglen == strlen ("message") && strncmp (msg, "message", msglen) == 0,
+        n = stdlog_decode (valid[i],
+                           strlen (valid[i]),
+                           &hdr,
+                           &sd,
+                           &sdlen,
+                           &msg,
+                           &msglen);
+        ok (n == 0
+            && msglen == strlen ("message")
+            && strncmp (msg, "message", msglen) == 0,
             "successfully decoded %s", valid[i]);
         i++;
     }

t/t0009-dmesg.t

@@ -108,6 +108,12 @@ test_expect_success 'logged non-ascii characters handled ok' '
 	/bin/echo -n -e "\xFF\xFE\x82\x00" | flux logger &&
 	flux dmesg
 '
+test_expect_success 'logged non-ascii printable characters are unmodified' '
+	flux logger ƒ Φ Ψ Ω Ö &&
+	flux dmesg | tail -1 > dmesg.utf-8 &&
+	test_debug "cat dmesg.utf-8" &&
+	grep "ƒ Φ Ψ Ω Ö" dmesg.utf-8
+'
 test_expect_success 'dmesg request with empty payload fails with EPROTO(71)' '
 	${RPC} log.dmesg 71 </dev/null
 '