Added support for UTF-8 during decoding.

The JSON specification does not require it, but some json encoders are using utf-8 encoding for json encoding.
Added related autotest case.

Author: jocelyn

library/kernel/json_string.e

@@ -61,128 +61,190 @@ feature -- Access
     item: STRING
             -- Contents with escaped entities if any
 
+feature -- Conversion
+
 	unescaped_string_8: STRING_8
-			-- Unescaped string from `item'
+			-- Unescaped string from `item'.
+			--| note: valid only if `item' does not encode any unicode character.
+		local
+			s: like item
+		do
+			s := item
+			create Result.make (s.count)
+			unescape_to_string_8 (Result)
+		end
+
+	unescaped_string_32: STRING_32
+			-- Unescaped string 32 from `item'
+			--| some encoders uses UTF-8 , and not the recommended pure json encoding
+			--| thus, let's support the UTF-8 encoding during decoding.
+		local
+			s: READABLE_STRING_8
+		do
+			s := item
+			create Result.make (s.count)
+			unescape_to_string_32 (Result)
+		end
+
+    representation: STRING
+			-- String representation of `item' with escaped entities if any
+        do
+            create Result.make (item.count + 2)
+            Result.append_character ('%"')
+            Result.append (item)
+            Result.append_character ('%"')
+        end
+
+	unescape_to_string_8 (a_output: STRING_8)
+			-- Unescape string `item' into `a_output'.
+			--| note: valid only if `item' does not encode any unicode character.
 		local
 			s: like item
 			i, n: INTEGER
 			c: CHARACTER
 		do
 			s := item
 			n := s.count
-			create Result.make (n)
 			from i := 1 until i > n loop
 				c := s[i]
 				if c = '\' then
 					if i < n then
 						inspect s[i+1]
 						when '\' then
-							Result.append_character ('\')
+							a_output.append_character ('\')
 							i := i + 2
 						when '%"' then
-							Result.append_character ('%"')
+							a_output.append_character ('%"')
 							i := i + 2
 						when 'b' then
-							Result.append_character ('%B')
+							a_output.append_character ('%B')
 							i := i + 2
 						when 'f' then
-							Result.append_character ('%F')
+							a_output.append_character ('%F')
 							i := i + 2
 						when 'n' then
-							Result.append_character ('%N')
+							a_output.append_character ('%N')
 							i := i + 2
 						when 'r' then
-							Result.append_character ('%R')
+							a_output.append_character ('%R')
 							i := i + 2
 						when 't' then
-							Result.append_character ('%T')
+							a_output.append_character ('%T')
 							i := i + 2
 						when 'u' then
 							--| Leave Unicode \uXXXX unescaped
-							Result.append_character ('\')
+							a_output.append_character ('\')
 							i := i + 1
 						else
-							Result.append_character ('\')
+							a_output.append_character ('\')
 							i := i + 1
 						end
 					else
-						Result.append_character ('\')
+						a_output.append_character ('\')
 						i := i + 1
 					end
 				else
-					Result.append_character (c)
+					a_output.append_character (c)
 					i := i + 1
 				end
 			end
 		end
 
-	unescaped_string_32: STRING_32
-			-- Unescaped string 32 from `item'
+	unescape_to_string_32 (a_output: STRING_32)
+			-- Unescape string `item' into `a_output' string 32.
+			--| some encoders uses UTF-8 , and not the recommended pure json encoding
+			--| thus, let's support the UTF-8 encoding during decoding.	
 		local
-			s: like item
+			s: READABLE_STRING_8
 			i, n: INTEGER
-			c: CHARACTER
-			hex: STRING
+			c: NATURAL_32
+			ch: CHARACTER_8
+			hex: READABLE_STRING_8
 		do
 			s := item
 			n := s.count
-			create Result.make (n)
 			from i := 1 until i > n loop
-				c := s[i]
-				if c = '\' then
+				ch := s.item (i)
+				if ch = '\' then
 					if i < n then
 						inspect s[i+1]
 						when '\' then
-							Result.append_character ('\')
+							a_output.append_character ('\')
 							i := i + 2
 						when '%"' then
-							Result.append_character ('%"')
+							a_output.append_character ('%"')
 							i := i + 2
 						when 'b' then
-							Result.append_character ('%B')
+							a_output.append_character ('%B')
 							i := i + 2
 						when 'f' then
-							Result.append_character ('%F')
+							a_output.append_character ('%F')
 							i := i + 2
 						when 'n' then
-							Result.append_character ('%N')
+							a_output.append_character ('%N')
 							i := i + 2
 						when 'r' then
-							Result.append_character ('%R')
+							a_output.append_character ('%R')
 							i := i + 2
-						when 'T' then
-							Result.append_character ('%T')
+						when 't' then
+							a_output.append_character ('%T')
 							i := i + 2
 						when 'u' then
-							hex := s.substring (i+2, i+2+4 - 1)
+							hex := s.substring (i + 2, i + 5) -- i+2 , i+2+4-1
 							if hex.count = 4 then
-								Result.append_code (hexadecimal_to_natural_32 (hex))
+								a_output.append_code (hexadecimal_to_natural_32 (hex))
 							end
-							i := i + 2 + 4
+							i := i + 6 -- i +2 +4
 						else
-							Result.append_character ('\')
+							a_output.append_character ('\')
 							i := i + 1
 						end
 					else
-						Result.append_character ('\')
+						a_output.append_character ('\')
 						i := i + 1
 					end
 				else
-					Result.append_character (c.to_character_32)
+					c := ch.natural_32_code
+					if c <= 0x7F then
+							-- 0xxxxxxx
+						check ch = c.to_character_32 end
+						a_output.append_character (ch)
+					elseif c <= 0xDF then
+							-- 110xxxxx 10xxxxxx
+						i := i + 1
+						if i <= n then
+							a_output.append_code (
+									((c & 0x1F) |<< 6) |
+									(s.code (i) & 0x3F)
+								)
+						end
+					elseif c <= 0xEF then
+							-- 1110xxxx 10xxxxxx 10xxxxxx
+						i := i + 2
+						if i <= n then
+							a_output.append_code (
+									((c & 0xF) |<< 12) |
+									((s.code (i - 1) & 0x3F) |<< 6) |
+									(s.code (i) & 0x3F)
+								)
+						end
+					elseif c <= 0xF7 then
+							-- 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
+						i := i + 3
+						if i <= n then
+							a_output.append_code (
+									((c & 0x7) |<< 18) |
+									((s.code (i - 2) & 0x3F) |<< 12) |
+									((s.code (i - 1) & 0x3F) |<< 6) |
+									(s.code (i) & 0x3F)
+								)
+						end
+					end
 					i := i + 1
 				end
 			end
 		end
 
-    representation: STRING
-			-- String representation of `item' with escaped entities if any
-        do
-            create Result.make (item.count + 2)
-            Result.append_character ('%"')
-            Result.append (item)
-            Result.append_character ('%"')
-        end
-
 feature -- Visitor pattern
 
     accept (a_visitor: JSON_VISITOR)
@@ -231,8 +293,18 @@ feature {NONE} -- Implementation
 
 	is_hexadecimal (s: READABLE_STRING_8): BOOLEAN
 			-- Is `s' an hexadecimal value?	
+		local
+			i: INTEGER
 		do
-			Result := across s as scur all scur.item.is_hexa_digit end
+			from
+				Result := True
+				i := 1
+			until
+				i > s.count or not Result
+			loop
+				Result := s[i].is_hexa_digit
+				i := i + 1
+			end
 		end
 
 	hexadecimal_to_natural_32 (s: READABLE_STRING_8): NATURAL_32

test/autotest/test_suite/test_json_suite.e

@@ -1,4 +1,4 @@
-note
+note
 	description: "[
 		Eiffel tests that can be executed by testing tool.
 	]"
@@ -62,6 +62,27 @@ feature -- Tests Pass
 			end
 		end
 
+	test_json_utf_8_pass1
+		local
+			parse_json: like new_json_parser
+			utf: UTF_CONVERTER
+			s: READABLE_STRING_32
+		do
+			s := {STRING_32} "{ %"nihaoma%": %"你好吗\t?%" }"
+
+			parse_json := new_json_parser (utf.string_32_to_utf_8_string_8 (s))
+			json_value := parse_json.parse_json
+			assert ("utf8.pass1.json", parse_json.is_parsed = True)
+			if
+				attached {JSON_OBJECT} json_value as jo and then
+				attached {JSON_STRING} jo.item ("nihaoma") as js
+			then
+				assert ("utf8.nihaoma", js.unescaped_string_32.same_string ({STRING_32} "你好吗%T?"))
+			else
+				assert ("utf8.nihaoma", False)
+			end
+		end
+
 feature -- Tests Failures
     test_json_fail1
     		--