Skip to content

Commit 4cff17a

Browse files
Aidan63Aidan Lee
Aidan63
and
Aidan Lee
authored
Marshal String Encodings (#1283)
* Add new encoding tests and implementation * Remove old marshal stuff and return an empty string instead of throwing * update tests * Case change * Update utf16 codepoint encoder * Add some extra functions * copyTo function for view * single bounds check for utf8 encode * marshal writes * remove some conversion issues * Remove un-needed cast * Remove questionable implicit view to pointer conversions * int returns * switch to a dedicated codepoint function for utf16 * move to a dedicated codepoint function for utf8 as well * Fix incorrect index reuse * Add new view extension cstring function tests * Add a smart strings guard * const view * const ref marshal read and writes * more const ref and less object copying * Fix index reuse issue * separate build xml for encoding * Don't return hx::Throw result in non smart string case * Revert "separate build xml for encoding" This reverts commit 695e134. * better smart string handling and shuffle some functions in lower hxcpp * Revert some version guards * Add pointer check define * I don't understand why this is needed No changes were made in this area, dev haxe + hxcpp also fails to build that test locally for me * remove addition since there must be something else weird going on * Could it really be this? * Dynamic char32_t support --------- Co-authored-by: Aidan Lee <[email protected]>
1 parent 596badd commit 4cff17a

File tree

25 files changed

+1809
-472
lines changed

25 files changed

+1809
-472
lines changed

‎.gitignore‎

Lines changed: 406 additions & 0 deletions
Large diffs are not rendered by default.

‎include/Array.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ template<> struct ReturnNull<float> { typedef Dynamic type; };
4040
template<> struct ReturnNull<bool> { typedef Dynamic type; };
4141
template<> struct ReturnNull<char> { typedef Dynamic type; };
4242
template<> struct ReturnNull<char16_t> { typedef Dynamic type; };
43+
template<> struct ReturnNull<char32_t> { typedef Dynamic type; };
4344
template<> struct ReturnNull<signed char> { typedef Dynamic type; };
4445
template<> struct ReturnNull<unsigned char> { typedef Dynamic type; };
4546
template<> struct ReturnNull<short> { typedef Dynamic type; };
@@ -433,6 +434,7 @@ template<> struct ArrayClassId<signed char> { enum { id=hx::clsIdArrayByte }; };
433434
template<> struct ArrayClassId<unsigned short> { enum { id=hx::clsIdArrayShort }; };
434435
template<> struct ArrayClassId<signed short> { enum { id=hx::clsIdArrayShort }; };
435436
template<> struct ArrayClassId<char16_t> { enum { id = hx::clsIdArrayShort }; };
437+
template<> struct ArrayClassId<char32_t> { enum { id = hx::clsIdArrayInt }; };
436438
template<> struct ArrayClassId<unsigned int> { enum { id=hx::clsIdArrayInt }; };
437439
template<> struct ArrayClassId<signed int> { enum { id=hx::clsIdArrayInt }; };
438440
template<> struct ArrayClassId<float> { enum { id=hx::clsIdArrayFloat32 }; };

‎include/Dynamic.h‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ class HXCPP_EXTERN_CLASS_ATTRIBUTES Dynamic : public hx::ObjectPtr<hx::Object>
2525
Dynamic(unsigned char inVal);
2626
Dynamic(signed char inVal);
2727
Dynamic(char16_t inVal);
28+
Dynamic(char32_t inVal);
2829
Dynamic(const cpp::CppInt32__ &inVal);
2930
Dynamic(bool inVal);
3031
Dynamic(double inVal);
@@ -73,6 +74,7 @@ class HXCPP_EXTERN_CLASS_ATTRIBUTES Dynamic : public hx::ObjectPtr<hx::Object>
7374
inline operator char () const { return mPtr ? mPtr->__ToInt() : 0; }
7475
inline operator signed char () const { return mPtr ? mPtr->__ToInt() : 0; }
7576
inline operator char16_t () const { return mPtr ? mPtr->__ToInt() : 0; }
77+
inline operator char32_t () const { return mPtr ? mPtr->__ToInt() : 0; }
7678
inline operator bool() const { return mPtr && mPtr->__ToInt(); }
7779
inline operator cpp::Int64() const { return mPtr ? mPtr->__ToInt64() : 0; }
7880
inline operator cpp::UInt64() const { return mPtr ? mPtr->__ToInt64() : 0; }
@@ -174,6 +176,7 @@ class HXCPP_EXTERN_CLASS_ATTRIBUTES Dynamic : public hx::ObjectPtr<hx::Object>
174176
bool operator op (signed char inRHS) const { return IsNumeric() && ((double)(*this) op (double)inRHS); } \
175177
bool operator op (unsigned char inRHS) const { return IsNumeric() && ((double)(*this) op (double)inRHS); } \
176178
bool operator op (char16_t inRHS) const { return IsNumeric() && ((double)(*this) op (double)inRHS); } \
179+
bool operator op (char32_t inRHS) const { return IsNumeric() && ((double)(*this) op (double)inRHS); } \
177180
bool operator op (bool inRHS) const { return IsBool() && ((double)(*this) op (double)inRHS); } \
178181

179182
bool operator != (const String &inRHS) const { return !mPtr || ((String)(*this) != inRHS); }
@@ -188,6 +191,7 @@ class HXCPP_EXTERN_CLASS_ATTRIBUTES Dynamic : public hx::ObjectPtr<hx::Object>
188191
bool operator != (signed char inRHS) const { return !IsNumeric() || ((double)(*this) != (double)inRHS); }
189192
bool operator != (unsigned char inRHS) const { return !IsNumeric() || ((double)(*this) != (double)inRHS); }
190193
bool operator != (char16_t inRHS) const { return !IsNumeric() || ((double)(*this) != (double)inRHS); }
194+
bool operator != (char32_t inRHS) const { return !IsNumeric() || ((double)(*this) != (double)inRHS); }
191195
bool operator != (bool inRHS) const { return !IsBool() || ((double)(*this) != (double)inRHS); }
192196

193197

@@ -234,6 +238,7 @@ class HXCPP_EXTERN_CLASS_ATTRIBUTES Dynamic : public hx::ObjectPtr<hx::Object>
234238
Dynamic operator+(const signed char &i) const;
235239
Dynamic operator+(const unsigned char &i) const;
236240
Dynamic operator+(const char16_t& i) const;
241+
Dynamic operator+(const char32_t& i) const;
237242
Dynamic operator+(const double &d) const;
238243
Dynamic operator+(const float &d) const;
239244
Dynamic operator+(const cpp::Variant &d) const;
@@ -277,6 +282,8 @@ class HXCPP_EXTERN_CLASS_ATTRIBUTES Dynamic : public hx::ObjectPtr<hx::Object>
277282
{ return mPtr->__GetType()==vtInt ? Dynamic((int)(*this) op inRHS) : Dynamic((double)(*this) op inRHS); } \
278283
Dynamic operator op (const char16_t &inRHS) const \
279284
{ return mPtr->__GetType()==vtInt ? Dynamic((int)(*this) op inRHS) : Dynamic((double)(*this) op inRHS); } \
285+
Dynamic operator op (const char32_t &inRHS) const \
286+
{ return mPtr->__GetType()==vtInt ? Dynamic((int)(*this) op inRHS) : Dynamic((double)(*this) op inRHS); } \
280287
Dynamic operator op (const cpp::Int64 &inRHS) const \
281288
{ return Dynamic((double)(*this) op inRHS); } \
282289
Dynamic operator op (const cpp::UInt64 &inRHS) const \
@@ -465,6 +472,7 @@ COMPARE_DYNAMIC_OP( > )
465472
inline double operator op (const signed char &inLHS,const Dynamic &inRHS) { return inLHS op (double)inRHS; } \
466473
inline double operator op (const unsigned char &inLHS,const Dynamic &inRHS) { return inLHS op (double)inRHS; } \
467474
inline double operator op (const char16_t &inLHS,const Dynamic &inRHS) { return inLHS op (double)inRHS; } \
475+
inline double operator op (const char32_t &inLHS,const Dynamic &inRHS) { return inLHS op (double)inRHS; } \
468476

469477
ARITH_DYNAMIC( - )
470478
ARITH_DYNAMIC( + )

‎include/cpp/Variant.h‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,7 @@ namespace cpp
110110
inline operator char () const { return asInt(); }
111111
inline operator signed char () const { return asInt(); }
112112
inline operator char16_t() const { return asInt(); }
113+
inline operator char32_t() const { return asInt(); }
113114
inline operator cpp::Int64 () const { return asInt64(); }
114115
inline operator cpp::UInt64 () const { return asInt64(); }
115116
inline bool operator !() const { return !asInt(); }
@@ -208,6 +209,7 @@ namespace cpp
208209
inline bool operator op (signed char inRHS) const { return isNumeric() && (asDouble() op (double)inRHS); } \
209210
inline bool operator op (unsigned char inRHS) const { return isNumeric() && (asDouble() op (double)inRHS); } \
210211
inline bool operator op (char16_t inRHS) const { return isNumeric() && (asDouble() op (double)inRHS); } \
212+
inline bool operator op (char32_t inRHS) const { return isNumeric() && (asDouble() op (double)inRHS); } \
211213
inline bool operator op (bool inRHS) const { return isBool() && (asDouble() op (double)inRHS); } \
212214
inline bool operator op (const Dynamic &inRHS) const { return Compare(inRHS) op 0; } \
213215

@@ -281,6 +283,7 @@ namespace cpp
281283
inline double operator op (const signed char &inLHS,const cpp::Variant &inRHS) { return inLHS op (double)inRHS; } \
282284
inline double operator op (const unsigned char &inLHS,const cpp::Variant &inRHS) { return inLHS op (double)inRHS; } \
283285
inline double operator op (const char16_t &inLHS,const cpp::Variant &inRHS) { return inLHS op (double)inRHS; } \
286+
inline double operator op (const char32_t &inLHS,const cpp::Variant &inRHS) { return inLHS op (double)inRHS; } \
284287
inline double operator op (const signed short &inLHS,const cpp::Variant &inRHS) { return inLHS op (double)inRHS; } \
285288
inline double operator op (const unsigned short &inLHS,const cpp::Variant &inRHS) { return inLHS op (double)inRHS; } \
286289
inline double operator op (const cpp::Int64 &inLHS,const cpp::Variant &inRHS) { return inLHS op (double)inRHS; } \
@@ -608,6 +611,7 @@ HX_VARIANT_OP_ISEQ(unsigned short)
608611
HX_VARIANT_OP_ISEQ(signed char)
609612
HX_VARIANT_OP_ISEQ(unsigned char)
610613
HX_VARIANT_OP_ISEQ(char16_t)
614+
HX_VARIANT_OP_ISEQ(char32_t)
611615
HX_VARIANT_OP_ISEQ(bool)
612616

613617
inline bool operator < (bool inLHS,const cpp::Variant &inRHS) { return false; }
@@ -639,6 +643,8 @@ inline bool operator > (bool inLHS,const cpp::Variant &inRHS) { return false; }
639643
{ return inRHS.isNumeric() && (inLHS op (double)inRHS); } \
640644
inline bool operator op (char16_t inLHS,const ::cpp::Variant &inRHS) \
641645
{ return inRHS.isNumeric() && (inLHS op (double)inRHS); } \
646+
inline bool operator op (char32_t inLHS,const ::cpp::Variant &inRHS) \
647+
{ return inRHS.isNumeric() && (inLHS op (double)inRHS); } \
642648
inline bool operator op (const null &,const ::cpp::Variant &inRHS) \
643649
{ return false; } \
644650

‎include/cpp/encoding/Ascii.hpp‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
#pragma once
2+
3+
namespace cpp
4+
{
5+
namespace encoding
6+
{
7+
struct Ascii final
8+
{
9+
static bool isEncoded(const String& string);
10+
11+
/// <summary>
12+
/// Encode the provided string to ASCII bytes and write them to the buffer.
13+
/// If the provided string is UTF16 encoded an exception is raised and nothing is written to the buffer.
14+
/// </summary>
15+
/// <returns>Number of chars written to the buffer.</returns>
16+
static int64_t encode(const String& string, cpp::marshal::View<uint8_t> buffer);
17+
18+
/// <summary>
19+
/// Create a string from the provided ASCII bytes.
20+
/// </summary>
21+
static String decode(cpp::marshal::View<uint8_t> string);
22+
};
23+
}
24+
}

‎include/cpp/encoding/Utf16.hpp‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
#pragma once
2+
3+
namespace cpp
4+
{
5+
namespace encoding
6+
{
7+
struct Utf16 final
8+
{
9+
static bool isEncoded(const String& string);
10+
11+
static int getByteCount(const char32_t& codepoint);
12+
static int64_t getByteCount(const String& string);
13+
14+
static int getCharCount(const char32_t& codepoint);
15+
static int64_t getCharCount(const String& string);
16+
17+
static int encode(const char32_t& codepoint, const cpp::marshal::View<uint8_t>& buffer);
18+
static int64_t encode(const String& string, const cpp::marshal::View<uint8_t>& buffer);
19+
20+
static char32_t codepoint(const cpp::marshal::View<uint8_t>& buffer);
21+
static String decode(const cpp::marshal::View<uint8_t>& buffer);
22+
};
23+
}
24+
}

‎include/cpp/encoding/Utf8.hpp‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
#pragma once
2+
3+
namespace cpp
4+
{
5+
namespace encoding
6+
{
7+
struct Utf8 final
8+
{
9+
static int getByteCount(const char32_t& codepoint);
10+
static int64_t getByteCount(const String& string);
11+
12+
static int getCharCount(const char32_t& codepoint);
13+
static int64_t getCharCount(const String& string);
14+
15+
static int encode(const char32_t& codepoint, const cpp::marshal::View<uint8_t>& buffer);
16+
static int64_t encode(const String& string, const cpp::marshal::View<uint8_t>& buffer);
17+
18+
static char32_t codepoint(const cpp::marshal::View<uint8_t>& buffer);
19+
static String decode(const cpp::marshal::View<uint8_t>& buffer);
20+
};
21+
}
22+
}

0 commit comments

Comments
 (0)