stdlib: Check for illegal surrogates when wchar_t is 2 bytes

Handling 2-byte wchar_t values requires extra care to avoid
illegal surrogate sequences both in and out of the library.

Make sure a high surrogate is always followed by a low surrogate
and that a low surrogate never stands alone.

When parsing utf-8 into 2-byte wchar_t, make sure the utf-8 value
doesn't contain a surrogate value.

Signed-off-by: Keith Packard <[email protected]>

Author: keith-packard

newlib/libc/stdlib/mbtowc_r.c

@@ -629,6 +629,12 @@ __utf8_mbtowc (
       tmp = (wchar_t)((state->__value.__wchb[0] & 0x0f) << 12)
 	|    (wchar_t)((state->__value.__wchb[1] & 0x3f) << 6)
 	|     (wchar_t)(ch & 0x3f);
+      /* Check for surrogates */
+      if (0xd800 <= tmp && tmp <= 0xdfff)
+        {
+          _REENT_ERRNO(r) = EILSEQ;
+          return -1;
+        }
       *pwc = tmp;
       return i;
     }

newlib/libc/stdlib/wctomb_r.c

@@ -55,6 +55,12 @@ __utf8_wctomb (
   if (sizeof (wchar_t) == 2 && state->__count == -4
       && (wchar < 0xdc00 || wchar > 0xdfff))
     {
+      /* Unexpected extra high surrogate */
+      if (0xd800 <= wchar && wchar <= 0xdbff)
+        {
+          _REENT_ERRNO(r) = EILSEQ;
+          return -1;
+        }
       /* There's a leftover lone high surrogate.  Write out the CESU-8 value
 	 of the surrogate and proceed to convert the given character.  Note
 	 to return extra 3 bytes. */
@@ -86,6 +92,12 @@ __utf8_wctomb (
 	  uint32_t tmp;
 	  if (wchar <= 0xdbff)
 	    {
+              if (state->__count == -4)
+                {
+                  /* Extra high surrogate */
+                  _REENT_ERRNO(r) = EILSEQ;
+                  return -1;
+                }
 	      /* First half of a surrogate pair.  Store the state and
 	         return ret + 0. */
 	      tmp = ((wchar & 0x3ff) << 10) + 0x10000;
@@ -110,7 +122,9 @@ __utf8_wctomb (
 	      *s   = 0x80 |  (tmp &     0x3f);
 	      return 4;
 	    }
-	  /* Otherwise translate into CESU-8 value. */
+          /* Unexpected second half */
+          _REENT_ERRNO(r) = EILSEQ;
+          return -1;
 	}
       *s++ = 0xe0 | ((wchar & 0xf000) >> 12);
       *s++ = 0x80 | ((wchar &  0xfc0) >> 6);