fix #1190 support to display packet content as UTF-8

Author: lotabout

configure.ac

@@ -45,6 +45,12 @@ AC_LBL_C_INIT(V_CCOPT, V_INCLS)
 #
 AC_CHECK_HEADERS(rpc/rpc.h rpc/rpcent.h)
 
+#
+# Check for wide character support for UTF-8 handling
+#
+AC_CHECK_HEADERS(wchar.h)
+AC_CHECK_FUNCS(wcwidth)
+
 # On Linux, if Autoconf version >= 2.72 and GNU C Library version >= 2.34,
 # uncomment AC_SYS_YEAR2038_RECOMMENDED to ensure time_t is Y2038-safe.
 # (Can be done by autogen.sh)

netdissect.h

@@ -231,6 +231,7 @@ struct netdissect_options {
   int ndo_Aflag;		/* print packet only in ASCII observing TAB,
 				 * LF, CR and SPACE as graphical chars
 				 */
+  int ndo_utf8;		/* interpret ASCII output as UTF-8 */
   int ndo_Hflag;		/* dissect 802.11s draft mesh standard */
   const char *ndo_protocol;	/* protocol */
   jmp_buf ndo_early_end;	/* jmp_buf for setjmp()/longjmp() */

print-ascii.c

@@ -44,6 +44,11 @@
 
 #include <stdio.h>
 
+#ifdef HAVE_WCHAR_H
+#include <wchar.h>
+#include <wctype.h>
+#endif
+
 #include "netdissect-ctype.h"
 
 #include "netdissect.h"
@@ -56,6 +61,71 @@
 #define HEXDUMP_HEXSTUFF_PER_LINE \
 		(HEXDUMP_HEXSTUFF_PER_SHORT * HEXDUMP_SHORTS_PER_LINE)
 
+#ifdef HAVE_WCHAR_H
+
+/*
+ * The blow is_utf8_printable is taken from ngrep
+ * 
+ * Check if a UTF-8 character sequence is printable using standard library functions.
+ * Returns the number of bytes in the UTF-8 character if printable, 0 otherwise.
+ * Also returns the display width (1 or 2 columns) via the width_out parameter.
+ *
+ * This uses mbrtowc() to convert multi-byte UTF-8 to wide char, then iswprint()
+ * to check if it's printable, and wcwidth() to get the display width.
+ */
+static int is_utf8_printable(const unsigned char *s, size_t max_len, int *width_out) {
+	if (!s || max_len == 0) return 0;
+
+	mbstate_t state = {0};
+	wchar_t wc;
+
+	size_t len = mbrtowc(&wc, (const char *)s, max_len, &state);
+
+	/* Check for errors and incomplete sequences */
+	if (len == (size_t)-1) {
+		/* Encoding error */
+		return 0;
+	}
+
+	if (len == (size_t)-2) {
+		/* Incomplete multi-byte sequence (need more bytes) */
+		return 0;
+	}
+
+	if (len == 0) {
+		/* Null character */
+		return 0;
+	}
+
+	/* Check if the wide character is printable */
+#if defined(_WIN32) || defined(_WIN64)
+	/* Windows iswprint() is too conservative - be more permissive for UTF-8 */
+	/* Accept any valid UTF-8 character that's not a control character */
+	int is_printable = iswprint(wc) ||
+		(wc >= 0x80 && wc < 0xD800) ||  /* Most of BMP except surrogates */
+		(wc >= 0xE000 && wc < 0x110000); /* Private use + supplementary planes */
+
+	/* But exclude actual control characters */
+	if (wc < 0x20 || (wc >= 0x7F && wc < 0xA0)) {
+		is_printable = 0;
+	}
+#else
+	int is_printable = iswprint(wc);
+#endif
+
+	if (is_printable) {
+		/* Get display width (1 for normal chars, 2 for wide chars like CJK, 0 for combining) */
+		int w = wcwidth(wc);
+		if (w < 0) w = 1;  /* Treat non-printable/control as width 1 */
+		/* Note: wcwidth returns 0 for combining characters, which is correct */
+		if (width_out) *width_out = w;
+		return (int)len;
+	}
+
+	return 0;
+}
+#endif
+
 void
 ascii_print(netdissect_options *ndo,
             const u_char *cp, u_int length)
@@ -71,28 +141,51 @@ ascii_print(netdissect_options *ndo,
 		truncated = TRUE;
 	}
 	ND_PRINT("\n");
-	while (length != 0) {
-		s = GET_U_1(cp);
-		cp++;
-		length--;
-		if (s == '\r') {
-			/*
-			 * Don't print CRs at the end of the line; they
-			 * don't belong at the ends of lines on UN*X,
-			 * and the standard I/O library will give us one
-			 * on Windows so we don't need to print one
-			 * ourselves.
-			 *
-			 * In the middle of a line, just print a '.'.
-			 */
-			if (length > 1 && GET_U_1(cp) != '\n')
-				ND_PRINT(".");
+
+	u_int remaining = length;
+
+	while (length > 0) {
+		int utf8_len, width;
+		int j;
+
+		utf8_len = 0;
+
+#ifdef HAVE_WCHAR_H
+		if (ndo->ndo_utf8) {
+			utf8_len = is_utf8_printable(cp, remaining, &width);
+		}
+#endif
+
+		if (utf8_len > 0) {
+			/* Valid printable UTF-8 character */
+			for (j = 0; j < utf8_len; j++)
+				ND_PRINT("%c", cp[j]);
+			cp += utf8_len;
+			length -= utf8_len;
+
 		} else {
-			if (!ND_ASCII_ISGRAPH(s) &&
-			    (s != '\t' && s != ' ' && s != '\n'))
-				ND_PRINT(".");
-			else
-				ND_PRINT("%c", s);
+			s = GET_U_1(cp);
+			cp++;
+			length--;
+			if (s == '\r') {
+				/*
+				 * Don't print CRs at the end of the line; they
+				 * don't belong at the ends of lines on UN*X,
+				 * and the standard I/O library will give us one
+				 * on Windows so we don't need to print one
+				 * ourselves.
+				 *
+				 * In the middle of a line, just print a '.'.
+				 */
+				if (length > 1 && GET_U_1(cp) != '\n')
+					ND_PRINT(".");
+			} else {
+				if (!ND_ASCII_ISGRAPH(s) &&
+						(s != '\t' && s != ' ' && s != '\n'))
+					ND_PRINT(".");
+				else
+					ND_PRINT("%c", s);
+			}
 		}
 	}
 	if (truncated)
@@ -104,52 +197,69 @@ hex_and_ascii_print_with_offset(netdissect_options *ndo, const char *indent,
 				const u_char *cp, u_int length, u_int offset)
 {
 	u_int caplength;
-	u_int i;
-	u_int s1, s2;
-	u_int nshorts;
+	u_int nbytes_unprinted;
+	u_int s1;
 	int truncated = FALSE;
 	char hexstuff[HEXDUMP_SHORTS_PER_LINE*HEXDUMP_HEXSTUFF_PER_SHORT+1], *hsp;
-	char asciistuff[ASCII_LINELENGTH+1], *asp;
+	char asciistuff[ASCII_LINELENGTH+1+4], *asp;
+	u_int utf8_bytes_to_skip = 0;
 
 	caplength = ND_BYTES_AVAILABLE_AFTER(cp);
 	if (length > caplength) {
 		length = caplength;
 		truncated = TRUE;
 	}
-	nshorts = length / sizeof(u_short);
-	i = 0;
+	nbytes_unprinted = 0;
 	hsp = hexstuff; asp = asciistuff;
-	while (nshorts != 0) {
+	while (length != 0) {
 		s1 = GET_U_1(cp);
+
+		// insert the leading space of short
+		if ((hsp - hexstuff) % HEXDUMP_HEXSTUFF_PER_SHORT == 0) {
+			(void)snprintf(hsp, sizeof(hexstuff) - (hsp - hexstuff), " ");
+			hsp++;
+		}
+
+		// add the byte
+		(void)snprintf(hsp, sizeof(hexstuff) - (hsp - hexstuff), "%02x", s1);
+		hsp += 2;
+
+		if (utf8_bytes_to_skip > 0) {
+			// only pad the new line
+			if (nbytes_unprinted == (asp - asciistuff)) {
+				*(asp++) = ' ';
+			}
+			utf8_bytes_to_skip--;
+		} else {
+			// try to add the display (utf8) chars
+#ifdef HAVE_WCHAR_H
+			utf8_bytes_to_skip = ndo->ndo_utf8 ? is_utf8_printable(cp, length, NULL) : 0;
+#endif
+			if (utf8_bytes_to_skip > 0) {
+				u_int j;
+				for (j=0; j<utf8_bytes_to_skip; j++) {
+					*(asp++) = (char)GET_U_1(cp+j);
+				}
+				utf8_bytes_to_skip --;
+			} else {
+				*(asp++) = (char)(ND_ASCII_ISGRAPH(s1) ? s1 : '.');
+			}
+		}
+
 		cp++;
-		s2 = GET_U_1(cp);
-		cp++;
-		(void)snprintf(hsp, sizeof(hexstuff) - (hsp - hexstuff),
-		    " %02x%02x", s1, s2);
-		hsp += HEXDUMP_HEXSTUFF_PER_SHORT;
-		*(asp++) = (char)(ND_ASCII_ISGRAPH(s1) ? s1 : '.');
-		*(asp++) = (char)(ND_ASCII_ISGRAPH(s2) ? s2 : '.');
-		i++;
-		if (i >= HEXDUMP_SHORTS_PER_LINE) {
+		nbytes_unprinted++;
+		if (nbytes_unprinted >= (HEXDUMP_SHORTS_PER_LINE * sizeof(u_short))) {
 			*hsp = *asp = '\0';
 			ND_PRINT("%s0x%04x: %-*s  %s",
 			    indent, offset, HEXDUMP_HEXSTUFF_PER_LINE,
 			    hexstuff, asciistuff);
-			i = 0; hsp = hexstuff; asp = asciistuff;
+			nbytes_unprinted = 0; hsp = hexstuff; asp = asciistuff;
 			offset += HEXDUMP_BYTES_PER_LINE;
 		}
-		nshorts--;
-	}
-	if (length & 1) {
-		s1 = GET_U_1(cp);
-		cp++;
-		(void)snprintf(hsp, sizeof(hexstuff) - (hsp - hexstuff),
-		    " %02x", s1);
-		hsp += 3;
-		*(asp++) = (char)(ND_ASCII_ISGRAPH(s1) ? s1 : '.');
-		++i;
+		length--;
 	}
-	if (i > 0) {
+
+	if (nbytes_unprinted > 0) {
 		*hsp = *asp = '\0';
 		ND_PRINT("%s0x%04x: %-*s  %s",
 		     indent, offset, HEXDUMP_HEXSTUFF_PER_LINE,
@@ -159,6 +269,7 @@ hex_and_ascii_print_with_offset(netdissect_options *ndo, const char *indent,
 		nd_trunc_longjmp(ndo);
 }
 
+
 void
 hex_and_ascii_print(netdissect_options *ndo, const char *indent,
 		    const u_char *cp, u_int length)

tcpdump.c

@@ -103,6 +103,7 @@ The Regents of the University of California.  All rights reserved.\n";
 #include <stdlib.h>
 #include <string.h>
 #include <limits.h>
+#include <locale.h>
 #ifdef _WIN32
 #include <windows.h>
 #else
@@ -668,6 +669,7 @@ show_remote_devices_and_exit(void)
 #define OPTION_LENGTHS			138
 #define OPTION_TIME_T_SIZE		139
 #define OPTION_SKIP			140
+#define OPTION_UTF8			141
 
 static const struct option longopts[] = {
 	{ "buffer-size", required_argument, NULL, 'B' },
@@ -712,6 +714,7 @@ static const struct option longopts[] = {
 	{ "time-t-size", no_argument, NULL, OPTION_TIME_T_SIZE },
 	{ "ip-oneline", no_argument, NULL, 'g' },
 	{ "skip", required_argument, NULL, OPTION_SKIP },
+	{ "utf8", no_argument, NULL, OPTION_UTF8 },
 	{ "version", no_argument, NULL, OPTION_VERSION },
 	{ NULL, 0, NULL, 0 }
 };
@@ -728,6 +731,12 @@ static const struct option longopts[] = {
 #define IMMEDIATE_MODE_USAGE ""
 #endif
 
+#ifdef HAVE_WCHAR_H
+#define DISPLAY_UTF8_USAGE "[ --utf8 ]"
+#else
+#define DISPLAY_UTF8_USAGE
+#endif
+
 #ifndef _WIN32
 /* Drop root privileges and chroot if necessary */
 static void
@@ -1631,6 +1640,8 @@ main(int argc, char **argv)
 	memset(ndo, 0, sizeof(*ndo));
 	ndo_set_function_pointers(ndo);
 
+	setlocale(LC_CTYPE, "");
+
 	cnt = -1;
 	device = NULL;
 	infile = NULL;
@@ -2094,6 +2105,10 @@ main(int argc, char **argv)
 			    optarg, NULL, 0, INT_MAX, 0);
 			break;
 
+		case OPTION_UTF8:
+			++ndo->ndo_utf8;
+			break;
+
 #ifdef HAVE_PCAP_SET_TSTAMP_PRECISION
 		case OPTION_TSTAMP_MICRO:
 			ndo->ndo_tstamp_precision = PCAP_TSTAMP_PRECISION_MICRO;
@@ -3507,5 +3522,5 @@ print_usage(FILE *f)
 "\t\t[ --time-stamp-precision precision ] [ --micro ] [ --nano ]\n");
 #endif
 	(void)fprintf(f,
-"\t\t" z_FLAG_USAGE "[ -Z user ] [ expression ]\n");
+"\t\t" DISPLAY_UTF8_USAGE z_FLAG_USAGE "[ -Z user ] [ expression ]\n");
 }