Merge branch 'development' into dev14-rebase2

ras0219-msft · ras0219-msft · commit 132c739c6f5f · 2014-09-26T19:01:57.000-07:00
diff --git a/Release/include/cpprest/asyncrt_utils.h b/Release/include/cpprest/asyncrt_utils.h
@@ -1,12 +1,12 @@
 /***
 * ==++==
 *
-* Copyright (c) Microsoft Corporation. All rights reserved. 
+* Copyright (c) Microsoft Corporation. All rights reserved.
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * http://www.apache.org/licenses/LICENSE-2.0
-* 
+*
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -30,7 +30,7 @@
 #if defined(_MSC_VER) && (_MSC_VER >= 1800)
 #include <ppltasks.h>
 namespace pplx = Concurrency;
-#else 
+#else
 #include "pplx/pplxtasks.h"
 #endif
 
diff --git a/Release/src/json/json_parsing.cpp b/Release/src/json/json_parsing.cpp
@@ -1,12 +1,12 @@
 /***
 * ==++==
 *
-* Copyright (c) Microsoft Corporation. All rights reserved. 
+* Copyright (c) Microsoft Corporation. All rights reserved.
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * http://www.apache.org/licenses/LICENSE-2.0
-* 
+*
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -31,14 +31,14 @@
 #include <array>
 
 #if defined(_MSC_VER)
-#pragma warning(disable : 4127) // allow expressions like while(true) pass 
+#pragma warning(disable : 4127) // allow expressions like while(true) pass
 #endif
 using namespace web;
 using namespace web::json;
 using namespace utility;
 using namespace utility::conversions;
 
-std::array<signed char,128> _hexval = {{ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 
+std::array<signed char,128> _hexval = {{ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
                                          -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
                                          -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
                                           0,  1,  2,  3,  4,  5,  6,  7,  8,  9, -1, -1, -1, -1, -1, -1,
@@ -74,7 +74,7 @@ template <typename CharType>
 class JSON_Parser
 {
 public:
-    JSON_Parser() 
+    JSON_Parser()
         : m_currentLine(1),
           m_eof(std::char_traits<CharType>::eof()),
           m_currentColumn(1),
@@ -171,15 +171,15 @@ class JSON_Parser
     void CreateToken(typename JSON_Parser<CharType>::Token& tk, typename Token::Kind kind, Location &start)
     {
         tk.kind = kind;
-        tk.start = start; 
+        tk.start = start;
         tk.string_val.clear();
     }
 
     void CreateToken(typename JSON_Parser<CharType>::Token& tk, typename Token::Kind kind)
     {
         tk.kind = kind;
-        tk.start.m_line = m_currentLine; 
-        tk.start.m_column = m_currentColumn; 
+        tk.start.m_line = m_currentLine;
+        tk.start.m_column = m_currentColumn;
         tk.string_val.clear();
     }
 
@@ -405,16 +405,16 @@ namespace
     }
 #endif
 
-    static double anystod(const char* str) 
+    static double anystod(const char* str)
     {
 #ifdef _MS_WINDOWS
         return _strtod_l(str, nullptr, utility::details::scoped_c_thread_locale::c_locale());
 #else
-        return strtod(str, nullptr); 
+        return strtod(str, nullptr);
 #endif
     }
-    static double anystod(const wchar_t* str) 
-    { 
+    static double anystod(const wchar_t* str)
+    {
 #ifdef _MS_WINDOWS
         return _wcstod_l(str, nullptr, utility::details::scoped_c_thread_locale::c_locale());
 #else
@@ -697,10 +697,20 @@ bool JSON_StringParser<CharType>::CompleteComment(typename JSON_Parser<CharType>
     return true;
 }
 
+void convert_append_unicode_code_unit(JSON_Parser<wchar_t>::Token &token, char16_t value)
+{
+    token.string_val.push_back(value);
+}
+void convert_append_unicode_code_unit(JSON_Parser<char>::Token &token, char16_t value)
+{
+    utf16string utf16(reinterpret_cast<utf16char *>(&value), 1);
+    token.string_val.append(::utility::conversions::utf16_to_utf8(utf16));
+}
+
 template <typename CharType>
 inline bool JSON_Parser<CharType>::handle_unescape_char(Token &token)
 {
-    // This function converts unescape character pairs (e.g. "\t") into their ASCII or UNICODE representations (e.g. tab sign)
+    // This function converts unescape character pairs (e.g. "\t") into their ASCII or Unicode representations (e.g. tab sign)
     // Also it handles \u + 4 hexadecimal digits
     CharType ch = NextCharacter();
     switch (ch)
@@ -731,7 +741,8 @@ inline bool JSON_Parser<CharType>::handle_unescape_char(Token &token)
             return true;
         case 'u':
         {
-            // A four-hexdigit unicode character
+            // A four-hexdigit Unicode character.
+            // Transform into a 16 bit code point.
             int decoded = 0;
             for (int i = 0; i < 4; ++i)
             {
@@ -755,8 +766,8 @@ inline bool JSON_Parser<CharType>::handle_unescape_char(Token &token)
             }
 
             // Construct the character based on the decoded number
-            ch = static_cast<CharType>(decoded & 0xFFFF);
-            token.string_val.push_back(ch);
+            convert_append_unicode_code_unit(token, static_cast<char16_t>(decoded));
+
             return true;
         }
         default:
@@ -966,14 +977,14 @@ std::unique_ptr<web::json::details::_Object> JSON_Parser<CharType>::_ParseObject
     auto obj = utility::details::make_unique<web::json::details::_Object>(g_keep_json_object_unsorted);
     auto& elems = obj->m_object.m_elements;
 
-    if ( tkn.kind != JSON_Parser<CharType>::Token::TKN_CloseBrace ) 
+    if ( tkn.kind != JSON_Parser<CharType>::Token::TKN_CloseBrace )
     {
         while ( true )
         {
             // State 1: New field or end of object, looking for field name or closing brace
 
             std::basic_string<CharType> fieldName;
-        
+
             switch ( tkn.kind )
             {
             case JSON_Parser<CharType>::Token::TKN_StringLiteral:
@@ -1038,7 +1049,7 @@ std::unique_ptr<web::json::details::_Array> JSON_Parser<CharType>::_ParseArray(t
 
     auto result = utility::details::make_unique<web::json::details::_Array>();
 
-    if ( tkn.kind != JSON_Parser<CharType>::Token::TKN_CloseBracket ) 
+    if ( tkn.kind != JSON_Parser<CharType>::Token::TKN_CloseBracket )
     {
         while ( true )
         {
@@ -1096,7 +1107,7 @@ std::unique_ptr<web::json::details::_Value> JSON_Parser<CharType>::_ParseValue(t
                 return std::move(value);
             }
 
-        case JSON_Parser<CharType>::Token::TKN_NumberLiteral:  
+        case JSON_Parser<CharType>::Token::TKN_NumberLiteral:
             {
                 auto value = utility::details::make_unique<web::json::details::_Number>(tkn.double_val);
                 GetNextToken(tkn);
diff --git a/Release/tests/functional/json/parsing_tests.cpp b/Release/tests/functional/json/parsing_tests.cpp
@@ -1,12 +1,12 @@
 /***
 * ==++==
 *
-* Copyright (c) Microsoft Corporation. All rights reserved. 
+* Copyright (c) Microsoft Corporation. All rights reserved.
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * http://www.apache.org/licenses/LICENSE-2.0
-* 
+*
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -202,13 +202,21 @@ TEST(string_t)
 
     str = json::value::parse(U("\"\\t\""));
     VERIFY_ARE_EQUAL(U("\t"), str.as_string());
+}
 
-    str = json::value::parse(U("\"\\u0041\""));
+TEST(escaped_unicode_string)
+{
+    auto str = json::value::parse(U("\"\\u0041\""));
     VERIFY_ARE_EQUAL(U("A"), str.as_string());
 
     str = json::value::parse(U("\"\\u004B\""));
     VERIFY_ARE_EQUAL(U("K"), str.as_string());
 
+    str = json::value::parse(U("\"\\u20AC\""));
+    // Euro sign as a hexidecmial UTF-8
+    const auto euro = to_string_t("\xE2\x82\xAC");
+    VERIFY_ARE_EQUAL(euro, str.as_string());
+
     VERIFY_PARSING_THROW(json::value::parse(U("\"\\u0klB\"")));
 }
 
@@ -565,7 +573,7 @@ TEST(keep_order_while_parsing)
     VERIFY_ARE_EQUAL(obj[U("a")].as_integer(), 4);
 }
 
-TEST(non_default_locale, "Ignore:Apple", "263")
+TEST(non_default_locale)
 {
     std::string originalLocale = setlocale(LC_ALL, nullptr);
 #ifdef _MS_WINDOWS
