lib: tutf8e: A tiny UTF-8 encoder for C

CMakeLists.txt

@@ -50,6 +50,7 @@ option(FLB_SMALL              "Optimise for small size"       No)
 option(FLB_COVERAGE           "Build with code-coverage"      No)
 option(FLB_JEMALLOC           "Build with Jemalloc support"   No)
 option(FLB_REGEX              "Build with Regex support"     Yes)
+option(FLB_UTF8_ENCODER       "Build with UTF8 encoding support" Yes)
 option(FLB_PARSER             "Build with Parser support"    Yes)
 option(FLB_TLS                "Build with SSL/TLS support"    No)
 option(FLB_BINARY             "Build executable binary"      Yes)
@@ -310,6 +311,11 @@ add_subdirectory(${FLB_PATH_LIB_MPACK} EXCLUDE_FROM_ALL)
 # Miniz (zip)
 add_subdirectory(${FLB_PATH_LIB_MINIZ} EXCLUDE_FROM_ALL)
 
+# tutf8e
+if(FLB_UTF8_ENCODER)
+  add_subdirectory(${FLB_PATH_LIB_TUTF8E} EXCLUDE_FROM_ALL)
+endif()
+
 # Chunk I/O
 FLB_OPTION(CIO_LIB_STATIC  ON)
 FLB_OPTION(CIO_LIB_SHARED  OFF)
@@ -558,6 +564,12 @@ if(FLB_REGEX)
   FLB_DEFINITION(FLB_HAVE_REGEX)
 endif()
 
+# tutf8e (UTF8 Encoding)
+# =====================
+if(FLB_UTF8_ENCODER)
+  FLB_DEFINITION(FLB_HAVE_UTF8_ENCODER)
+endif()
+
 # LuaJIT (Scripting Support)
 # ==========================
 if(FLB_LUAJIT)

cmake/headers.cmake

@@ -19,6 +19,10 @@ include_directories(
   ${CMAKE_CURRENT_BINARY_DIR}/include
   )
 
+if(FLB_UTF8_ENCODER)
+  include_directories(${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_TUTF8E}/include)
+endif()
+
 # On Windows, the core uses libevent
 if(CMAKE_SYSTEM_NAME MATCHES "Windows")
   include_directories(

cmake/libraries.cmake

@@ -11,3 +11,4 @@ set(FLB_PATH_LIB_SQLITE  "lib/sqlite-amalgamation-3240000")
 set(FLB_PATH_LIB_ONIGMO  "lib/onigmo")
 set(FLB_PATH_LIB_MPACK   "lib/mpack-amalgamation-1.0")
 set(FLB_PATH_LIB_MINIZ   "lib/miniz")
+set(FLB_PATH_LIB_TUTF8E  "lib/tutf8e")

lib/tutf8e/CMakeLists.txt

@@ -0,0 +1,10 @@
+cmake_minimum_required(VERSION 2.8)
+project(tutf8e)
+
+set(CMAKE_C_FLAGS "-Os -Wall")
+
+include_directories(include)
+add_library(tutf8e STATIC src/tutf8e.c)
+
+add_executable(tutf8e-test test/test.c)
+target_link_libraries(tutf8e-test tutf8e)

lib/tutf8e/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2019 Nigel Stewart
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

lib/tutf8e/README.md

@@ -0,0 +1,84 @@
+# tutf8e
+
+  *Tute Feighty*
+
+  A tiny UTF-8 encoder for C.
+
+## Goals
+
+  * As small and fast as possible
+  * Narrowly scoped to one-step UTF-8 encoding in C
+  * Link only what you need and use
+  * MIT licence
+
+## Supported Encodings
+
+  * [iso-8859-1](https://en.wikipedia.org/wiki/ISO/IEC_8859-1) Latin-1 Western European
+  * [iso-8859-2](https://en.wikipedia.org/wiki/ISO/IEC_8859-2) Latin-2 East European
+  * [iso-8859-3](https://en.wikipedia.org/wiki/ISO/IEC_8859-3) Latin-3 South European
+  * [iso-8859-4](https://en.wikipedia.org/wiki/ISO/IEC_8859-4) Latin-4 North European
+  * [iso-8859-5](https://en.wikipedia.org/wiki/ISO/IEC_8859-5) Part 5: Latin/Cyrillic
+  * [iso-8859-6](https://en.wikipedia.org/wiki/ISO/IEC_8859-6) Part 6: Latin/Arabic
+  * [iso-8859-7](https://en.wikipedia.org/wiki/ISO/IEC_8859-7) Part 7: Latin/Greek
+  * [iso-8859-8](https://en.wikipedia.org/wiki/ISO/IEC_8859-8) Part 8: Latin/Hebrew
+  * [iso-8859-9](https://en.wikipedia.org/wiki/ISO/IEC_8859-9) Latin-5 Turkish
+  * [iso-8859-10](https://en.wikipedia.org/wiki/ISO/IEC_8859-10) Latin-6 Nordic
+  * [iso-8859-11](https://en.wikipedia.org/wiki/ISO/IEC_8859-11) Part 11: Latin/Thai
+  * [iso-8859-13](https://en.wikipedia.org/wiki/ISO/IEC_8859-13) Latin-7 Baltic Rim
+  * [iso-8859-14](https://en.wikipedia.org/wiki/ISO/IEC_8859-14) Latin-8 Celtic
+  * [iso-8859-15](https://en.wikipedia.org/wiki/ISO/IEC_8859-15) Latin-9 Western European 
+  * [iso-8859-16](https://en.wikipedia.org/wiki/ISO/IEC_8859-16) Latin-10 South-Eastern European
+  * [windows-1250](https://en.wikipedia.org/wiki/Windows-1250) Central European and Eastern European
+  * [windows-1251](https://en.wikipedia.org/wiki/Windows-1251) Cyrillic
+  * [windows-1252](https://en.wikipedia.org/wiki/Windows-1252) English
+  * [windows-1253](https://en.wikipedia.org/wiki/Windows-1253) Greek
+  * [windows-1254](https://en.wikipedia.org/wiki/Windows-1254) Turkish
+  * [windows-1255](https://en.wikipedia.org/wiki/Windows-1255) Hebrew
+  * [windows-1256](https://en.wikipedia.org/wiki/Windows-1256) Arabic
+  * [windows-1257](https://en.wikipedia.org/wiki/Windows-1257) Baltic
+  * [windows-1258](https://en.wikipedia.org/wiki/Windows-1258) Vietnamese
+
+## Test Procedure
+
+```
+$ ./codegen.py
+
+$ gcc src/* test/test.c -Iinclude
+
+$ ./a.out
+A quick brown fox jumps over the lazy dog
+Nechť již hříšné saxofony ďáblů rozezvučí síň úděsnými tóny waltzu, tanga a quickstepu.
+Pijamalı hasta yağız şoföre çabucak güvendi.
+Põdur Zagrebi tšellomängija-följetonist Ciqo külmetas kehvas garaažis
+В чащах юга жил бы цитрус? Да, но фальшивый экземпляр!
+διαφυλάξτε γενικά τη ζωή σας από βαθειά ψυχικά τραύματα
+עטלף אבק נס דרך מזגן שהתפוצץ כי חם
+Pijamalı hasta yağız şoföre çabucak güvendi.
+Flygande bäckasiner söka hwila på mjuka tuvor.
+เป็นมนุษย์สุดประเสริฐเลิศคุณค่า กว่าบรรดาฝูงสัตว์เดรัจฉาน จงฝ่าฟันพัฒนาวิชาการ อย่าล้างผลาญฤๅเข่นฆ่าบีฑาใคร ไม่ถือโทษโกรธแช่งซัดฮึดฮัดด่า หัดอภัยเหมือนกีฬาอัชฌาสัย ปฏิบัติประพฤติกฎกำหนดใจ พูดจาให้จ๊ะๆ จ๋าๆ น่าฟังเอยฯ
+Jeżu klątw, spłódź Finom część gry hańb!
+11 passed, 0 failed tests
+```
+
+## How small is it?
+
+512 bytes + overhead per encoding.
+
+```
+$ for i in src/*; do gcc -c $i -O1; done
+$ du -bhc *.o | grep total
+32K total
+
+$ for i in src/*; do gcc -c $i -O3; done
+$ du -bhc *.o | grep total
+32K total
+
+$ for i in src/*; do gcc -c $i -Os; done
+$ du -bhc *.o | grep total
+28K total
+```
+
+## Related
+
+  * [iconv](https://www.gnu.org/software/libiconv/)
+  * [icu](http://site.icu-project.org/)