Fix Formatter UTF8 encoding (#2944)

Author: mikee47

Sming/Core/Data/Format/Formatter.cpp

@@ -58,14 +58,14 @@ unsigned escapeControls(String& value, Options options)
 		if(escapeChar(c, options)) {
 			extra += 1; // "\"
 		} else if(options[Option::unicode]) {
-			if(uint8_t(c) < 0x20 || (c & 0x80)) {
+			if(uint8_t(c) < 0x20) {
 				extra += 5; // "\uNNNN"
 			}
 		} else if(uint8_t(c) < 0x20) {
 			extra += 3; // "\xnn"
 		} else if((c & 0x80) && options[Option::utf8]) {
-			// Characters such as £ (0xa3) are escaped to 0xc2 0xa3 in UTF-8
-			extra += 1; // 0xc2
+			// Characters from U+0080 to U+07FF are encoded in two bytes in UTF-8
+			extra += 1;
 		}
 	}
 	if(extra == 0) {
@@ -86,7 +86,7 @@ unsigned escapeControls(String& value, Options options)
 			*out++ = '\\';
 			c = esc;
 		} else if(options[Option::unicode]) {
-			if(uint8_t(c) < 0x20 || (c & 0x80)) {
+			if(uint8_t(c) < 0x20) {
 				*out++ = '\\';
 				*out++ = 'u';
 				*out++ = '0';
@@ -100,7 +100,8 @@ unsigned escapeControls(String& value, Options options)
 			*out++ = hexchar(uint8_t(c) >> 4);
 			c = hexchar(uint8_t(c) & 0x0f);
 		} else if((c & 0x80) && options[Option::utf8]) {
-			*out++ = 0xc2;
+			*out++ = 0xc0 | (c >> 6);
+			c = 0x80 | (c & 0x3f);
 		}
 		*out++ = c;
 	}

Sming/Core/Data/Format/Json.cpp

@@ -18,11 +18,12 @@ namespace Format
 Json json;
 
 /*
- * Check for invalid characters and replace them - can break browser
- * operation otherwise.
+ * JSON requires control characters, quotes and reverse solidus (backslash) to be escaped.
  *
- * This can occur if filenames become corrupted, so here we just
- * substitute an underscore _ for anything which fails to match UTF8.
+ * All other codepoints from 0x20 to 0xff are left unchanged.
+ * This is typically UTF8 but it could be binary or some other application-defined encoding.
+ *
+ * Therefore no validation is performed on the data.
  */
 void Json::escape(String& value) const
 {

tests/HostTests/modules/Formatter.cpp

@@ -17,7 +17,7 @@ class FormatterTest : public TestGroup
 		TEST_CASE("JSON")
 		{
 			DEFINE_FSTR_LOCAL(text1b, "A JSON\\ntest string\\twith escapes\\u0012\\u0000\\n"
-									  "Worth \\\"maybe\\\" \\u00a3 0.53. Yen \\u00a5 5bn.")
+									  "Worth \\\"maybe\\\" \xa3 0.53. Yen \xa5 5bn.")
 
 			Serial << text1 << endl;
 			String s(text1);