uchar: Add uchar implementation

This provides the C23 uchar.h and associated functions for translating
between UTF-8, UTF-16, UTF-32 and C multibyte encodings.

Signed-off-by: Keith Packard <[email protected]>

Author: keith-packard

newlib/libc/CMakeLists.txt

@@ -51,5 +51,6 @@ add_subdirectory(stdlib)
 add_subdirectory(string)
 add_subdirectory(time)
 add_subdirectory(tinystdio)
+add_subdirectory(uchar)
 add_subdirectory(xdr)
 add_subdirectory(locale)

newlib/libc/include/CMakeLists.txt

@@ -92,6 +92,7 @@ picolibc_headers(""
   termios.h
   threads.h
   time.h
+  uchar.h
   unctrl.h
   unistd.h
   utime.h

newlib/libc/include/meson.build

@@ -91,6 +91,7 @@ inc_headers = [
   'termios.h',
   'threads.h',
   'time.h',
+  'uchar.h',
   'unctrl.h',
   'unistd.h',
   'utime.h',

newlib/libc/include/sys/_types.h

@@ -209,6 +209,7 @@ typedef struct
   {
     wint_t __wch;
     unsigned char __wchb[4];
+    __uint32_t __ucs;
   } __value;		/* Value so far.  */
 } _mbstate_t;
 #endif

newlib/libc/include/uchar.h

@@ -0,0 +1,75 @@
+/*
+ * SPDX-License-Identifier: BSD-3-Clause
+ *
+ * Copyright © 2024 Keith Packard
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above
+ *    copyright notice, this list of conditions and the following
+ *    disclaimer in the documentation and/or other materials provided
+ *    with the distribution.
+ *
+ * 3. Neither the name of the copyright holder nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ * OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _UCHAR_H_
+#define _UCHAR_H_
+
+#include <sys/cdefs.h>
+#include <machine/_default_types.h>
+#include <sys/_types.h>
+#define __need_size_t
+#include <stddef.h>
+
+#define __STDC_VERSION_UCHAR_H_ 202311L
+
+_BEGIN_STD_C
+
+#ifndef _MBSTATE_DECLARED
+typedef _mbstate_t mbstate_t;
+#define _MBSTATE_DECLARED
+#endif
+
+typedef unsigned char char8_t;
+typedef __uint_least16_t char16_t;
+typedef __uint_least32_t char32_t;
+
+size_t mbrtoc8(char8_t * __restrict pc8, const char * __restrict s, size_t n,
+               _mbstate_t * __restrict ps);
+
+size_t c8rtomb(char * __restrict s, char8_t c8, _mbstate_t * __restrict ps);
+
+size_t mbrtoc16(char16_t * __restrict pc16, const char * __restrict s, size_t n,
+                _mbstate_t * __restrict ps);
+
+size_t c16rtomb(char * __restrict s, char16_t c16, _mbstate_t * __restrict ps);
+
+size_t mbrtoc32(char32_t * __restrict pc32, const char * __restrict s, size_t n,
+                _mbstate_t * __restrict ps);
+
+size_t c32rtomb(char * __restrict s, char32_t c32, _mbstate_t * __restrict ps);
+
+_END_STD_C
+
+#endif /* _UCHAR_H_ */

newlib/libc/meson.build

@@ -35,7 +35,7 @@
 
 libdirs = ['argz', 'ctype', 'errno', 'iconv', 'misc',
 	   'posix', 'search', 'signal', 'ssp', 'stdlib',
-	   'string', 'time', 'xdr', 'locale']
+	   'string', 'time', 'xdr', 'locale', 'uchar']
 
 if enable_picolib
   libdirs += 'picolib'

newlib/libc/uchar/CMakeLists.txt

@@ -0,0 +1,42 @@
+#
+# SPDX-License-Identifier: BSD-3-Clause
+#
+# Copyright © 2024 Keith Packard
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above
+#    copyright notice, this list of conditions and the following
+#    disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its
+#    contributors may be used to endorse or promote products derived
+#    from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+# INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+# OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+picolibc_sources(
+  c16rtomb.c
+  c32rtomb.c
+  c8rtomb.c
+  mbrtoc16.c
+  mbrtoc32.c
+  mbrtoc8.c
+  )

newlib/libc/uchar/c16rtomb.c

@@ -0,0 +1,75 @@
+/*
+ * SPDX-License-Identifier: BSD-3-Clause
+ *
+ * Copyright © 2024 Keith Packard
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above
+ *    copyright notice, this list of conditions and the following
+ *    disclaimer in the documentation and/or other materials provided
+ *    with the distribution.
+ *
+ * 3. Neither the name of the copyright holder nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ * OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "uchar-local.h"
+
+size_t
+c16rtomb (char *s, char16_t c16, mbstate_t *ps)
+{
+    static NEWLIB_THREAD_LOCAL mbstate_t local_state;
+
+    if (ps == NULL)
+        ps = &local_state;
+
+#if __SIZEOF_WCHAR_T__ == 2
+    return wcrtomb(s, (wchar_t) c16, ps);
+#elif __SIZEOF_WCHAR_T__ == 4
+    char32_t    c32;
+
+    /* High surrogate */
+    if (char16_is_high_surrogate(c16)) {
+        if (ps->__count != 0) {
+            errno = EILSEQ;
+            return (size_t) -1;
+        }
+        ps->__value.__ucs = ((char32_t) (c16 & 0x3ff) << 10) | 0x10000;
+        ps->__count = -1;
+        return 0;
+    } else if (char16_is_low_surrogate(c16)) {
+        if (ps->__count == -1) {
+            c32 = ps->__value.__ucs | (c16 & 0x3ff);
+            ps->__count = 0;
+        } else {
+            errno = EILSEQ;
+            return (size_t) -1;
+        }
+    } else {
+        c32 = c16;
+    }
+    return wcrtomb(s, (wchar_t) c32, ps);
+#else
+#error wchar_t size unknown
+#endif
+}

newlib/libc/uchar/c32rtomb.c

@@ -0,0 +1,61 @@
+/*
+ * SPDX-License-Identifier: BSD-3-Clause
+ *
+ * Copyright © 2024 Keith Packard
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above
+ *    copyright notice, this list of conditions and the following
+ *    disclaimer in the documentation and/or other materials provided
+ *    with the distribution.
+ *
+ * 3. Neither the name of the copyright holder nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ * OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "uchar-local.h"
+
+size_t
+c32rtomb (char *s, char32_t c32, mbstate_t *ps)
+{
+    static NEWLIB_THREAD_LOCAL mbstate_t local_state;
+
+    if (ps == NULL)
+        ps = &local_state;
+
+    if (!char32_is_valid(c32)) {
+        errno = EILSEQ;
+        return (size_t) -1;
+    }
+#if __SIZEOF_WCHAR_T__ == 2
+    if (char32_needs_surrogates(c32)) {
+        const wchar_t wc[2] = {
+            [0] = ((c32 - 0x10000) >> 10) + HIGH_SURROGATE_FIRST,
+            [1] = (c32 & 0x3ff) + LOW_SURROGATE_FIRST,
+        };
+        const wchar_t *wcp = wc;
+        return wcsnrtombs(s, &wcp, 2, SIZE_MAX, ps);
+    }
+#endif
+    return wcrtomb(s, (wchar_t) c32, ps);
+}