Merge pull request #119 from hanickadot/u8string

add std::u8string (with utf8 code point iterator)

Author: hanickadot

include/ctre-unicode.hpp

@@ -1,6 +1,7 @@
 #ifndef CTRE_V2__CTRE_UNICODE__HPP
 #define CTRE_V2__CTRE_UNICODE__HPP
 
+#include "ctre.hpp"
 #include "unicode-db.hpp"
 
 #endif

include/ctre/evaluation.hpp

@@ -37,34 +37,32 @@ template <size_t Limit> constexpr CTRE_FORCE_INLINE bool less_than(size_t i) {
 	}
 }
 
+template <typename ResultIterator, typename Pattern> using return_type = decltype(regex_results(std::declval<ResultIterator>(), find_captures(Pattern{})));
+
 // calling with pattern prepare stack and triplet of iterators
-template <typename Iterator, typename EndIterator, typename Pattern> 
-constexpr inline auto match_re(const Iterator begin, const EndIterator end, Pattern pattern) noexcept {
-	using return_type = decltype(regex_results(std::declval<Iterator>(), find_captures(pattern)));
-	return evaluate(begin, begin, end, return_type{}, ctll::list<start_mark, Pattern, assert_end, end_mark, accept>());
+template <typename Iterator, typename EndIterator, typename Pattern, typename ResultIterator = Iterator> 
+constexpr inline auto match_re(const Iterator begin, const EndIterator end, Pattern) noexcept {
+	return evaluate(begin, begin, end, return_type<ResultIterator, Pattern>{}, ctll::list<start_mark, Pattern, assert_end, end_mark, accept>());
 }
 
-template <typename Iterator, typename EndIterator, typename Pattern> 
-constexpr inline auto starts_with_re(const Iterator begin, const EndIterator end, Pattern pattern) noexcept {
-	using return_type = decltype(regex_results(std::declval<Iterator>(), find_captures(pattern)));
-	return evaluate(begin, begin, end, return_type{}, ctll::list<start_mark, Pattern, end_mark, accept>());
+template <typename Iterator, typename EndIterator, typename Pattern, typename ResultIterator = Iterator> 
+constexpr inline auto starts_with_re(const Iterator begin, const EndIterator end, Pattern) noexcept {
+	return evaluate(begin, begin, end, return_type<ResultIterator, Pattern>{}, ctll::list<start_mark, Pattern, end_mark, accept>());
 }
 
-template <typename Iterator, typename EndIterator, typename Pattern> 
-constexpr inline auto search_re(const Iterator begin, const EndIterator end, Pattern pattern) noexcept {
-	using return_type = decltype(regex_results(std::declval<Iterator>(), find_captures(pattern)));
-	
+template <typename Iterator, typename EndIterator, typename Pattern, typename ResultIterator = Iterator> 
+constexpr inline auto search_re(const Iterator begin, const EndIterator end, Pattern) noexcept {
 	constexpr bool fixed = starts_with_anchor(ctll::list<Pattern>{});
 
 	auto it = begin;
 	for (; end != it && !fixed; ++it) {
-		if (auto out = evaluate(begin, it, end, return_type{}, ctll::list<start_mark, Pattern, end_mark, accept>())) {
+		if (auto out = evaluate(begin, it, end, return_type<ResultIterator, Pattern>{}, ctll::list<start_mark, Pattern, end_mark, accept>())) {
 			return out;
 		}
 	}
 
 	// in case the RE is empty or fixed
-	return evaluate(begin, it, end, return_type{}, ctll::list<start_mark, Pattern, end_mark, accept>());
+	return evaluate(begin, it, end, return_type<ResultIterator, Pattern>{}, ctll::list<start_mark, Pattern, end_mark, accept>());
 }
 
 
@@ -109,7 +107,7 @@ constexpr CTRE_FORCE_INLINE R evaluate(const Iterator, Iterator current, const E
 
 template <typename R, typename Iterator, typename EndIterator, typename CharacterLike, typename... Tail, typename = std::enable_if_t<(MatchesCharacter<CharacterLike>::template value<decltype(*std::declval<Iterator>())>)>> 
 constexpr CTRE_FORCE_INLINE R evaluate(const Iterator begin, Iterator current, const EndIterator end, R captures, ctll::list<CharacterLike, Tail...>) noexcept {
-	if (end == current) return not_matched;
+	if (current == end) return not_matched;
 	if (!CharacterLike::match_char(*current)) return not_matched;
 	return evaluate(begin, current+1, end, captures, ctll::list<Tail...>());
 }

include/ctre/id.hpp

@@ -7,13 +7,15 @@ namespace ctre {
 
 template <auto... Name> struct id {
 	static constexpr auto name = ctll::fixed_string<sizeof...(Name)>{{Name...}};
-};
 
-template <auto... Name> constexpr auto operator==(id<Name...>, id<Name...>) noexcept -> std::true_type { return {}; }
+	friend constexpr auto operator==(id<Name...>, id<Name...>) noexcept -> std::true_type { return {}; }
 
-template <auto... Name1, auto... Name2> constexpr auto operator==(id<Name1...>, id<Name2...>) noexcept -> std::false_type { return {}; }
+	template <auto... Other> friend constexpr auto operator==(id<Name...>, id<Other...>) noexcept -> std::false_type { return {}; }
 
-template <auto... Name, typename T> constexpr auto operator==(id<Name...>, T) noexcept -> std::false_type { return {}; }
+	template <typename T> friend constexpr auto operator==(id<Name...>, T) noexcept -> std::false_type { return {}; }
+
+	template <typename T> friend constexpr auto operator==(T, id<Name...>) noexcept -> std::false_type { return {}; }
+};
 
 }
 

include/ctre/return_type.hpp

@@ -2,6 +2,7 @@
 #define CTRE__RETURN_TYPE__HPP
 
 #include "id.hpp"
+#include "utf8.hpp"
 #include <type_traits>
 #include <tuple>
 #include <string_view>
@@ -56,27 +57,46 @@ template <size_t Id, typename Name = void> struct captured_content {
 		}
 
 		constexpr CTRE_FORCE_INLINE const auto * data() const noexcept {
+			#if __cpp_char8_t >= 201811
+			if constexpr (std::is_same_v<Iterator, utf8_iterator>) {
+				return _begin.ptr;
+			} else {
+				return &*_begin;
+			}
+			#else
 			return &*_begin;
+			#endif
 		}
 
 		constexpr CTRE_FORCE_INLINE auto size() const noexcept {
-			return static_cast<size_t>(std::distance(_begin, _end));
+			return static_cast<size_t>(std::distance(begin(), end()));
+		}
+		
+		constexpr CTRE_FORCE_INLINE size_t unit_size() const noexcept {
+			#if __cpp_char8_t >= 201811
+			if constexpr (std::is_same_v<Iterator, utf8_iterator>) {
+				return static_cast<size_t>(std::distance(_begin.ptr, _end.ptr));
+			}
+			#endif
+			return static_cast<size_t>(std::distance(begin(), end()));
 		}
 
 		constexpr CTRE_FORCE_INLINE auto to_view() const noexcept {
-			return std::basic_string_view<char_type>(&*_begin, static_cast<size_t>(std::distance(_begin, _end)));
+			// TODO make sure we are working with contiguous range
+			return std::basic_string_view<char_type>(data(), static_cast<size_t>(unit_size()));
 		}
 
 		constexpr CTRE_FORCE_INLINE auto to_string() const noexcept {
-			return std::basic_string<char_type>(begin(), end());
+			// TODO make sure we are working with contiguous range
+			return std::basic_string<char_type>(data(), static_cast<size_t>(unit_size()));
 		}
 
 		constexpr CTRE_FORCE_INLINE auto view() const noexcept {
-			return std::basic_string_view<char_type>(&*_begin, static_cast<size_t>(std::distance(_begin, _end)));
+			return to_view();
 		}
 
 		constexpr CTRE_FORCE_INLINE auto str() const noexcept {
-			return std::basic_string<char_type>(begin(), end());
+			return to_string();
 		}
 
 		constexpr CTRE_FORCE_INLINE operator std::basic_string_view<char_type>() const noexcept {

include/ctre/utf8.hpp

@@ -0,0 +1,147 @@
+#ifndef CTRE__UTF8__HPP
+#define CTRE__UTF8__HPP
+
+#if __cpp_char8_t >= 201811
+
+#include "utility.hpp"
+#include <iterator>
+
+namespace ctre {
+
+struct utf8_iterator {
+	using self_type = utf8_iterator;
+	using value_type = char8_t;
+	using reference = char8_t;
+	using pointer = const char8_t *;
+	using iterator_category = std::forward_iterator_tag;
+	using difference_type = int;
+	
+	struct sentinel {
+
+	};
+	
+	const char8_t * ptr{nullptr};
+	const char8_t * end{nullptr};
+	
+	constexpr friend bool operator!=(const utf8_iterator & lhs, sentinel) {
+		return lhs.ptr < lhs.end;
+	}
+	
+	constexpr friend bool operator!=(const utf8_iterator & lhs, const utf8_iterator & rhs) {
+		return lhs.ptr != rhs.ptr;
+	}
+	
+	constexpr friend bool operator==(const utf8_iterator & lhs, sentinel) {
+		return lhs.ptr >= lhs.end;
+	}
+	
+	constexpr utf8_iterator & operator=(const char8_t * rhs) {
+		ptr = rhs;
+		return *this;
+	}
+	
+	constexpr operator const char8_t *() const noexcept {
+		return ptr;
+	}
+	
+	constexpr utf8_iterator & operator++() noexcept {
+		// the contant is mapping from first 5 bits of first code unit to length of UTF8 code point -1
+		// xxxxx -> yy (5 bits to 2 bits)
+		// 5 bits are 32 combination, and for each I need 2 bits, hence 64 bit constant
+		// (*ptr >> 3)  looks at left 5 bits
+		// << 1  will multiply it by 2
+		// & 0b11u  selects only needed two bits
+		// +1  because each iteration is at least one code unit forward
+		
+		ptr += ((0x3A55000000000000ull >> ((*ptr >> 3) << 1)) & 0b11u) + 1;
+		return *this;
+	}
+	
+	constexpr utf8_iterator operator+(unsigned step) const noexcept {
+		utf8_iterator result = *this;
+		while (step > 0) {
+			++result;
+			step--;
+		}
+		return result;
+	}
+	
+	constexpr char32_t operator*() const noexcept {
+		constexpr uint64_t lengths = 0x3A55000000000000ull;
+		constexpr char32_t mojibake = 0xFFFDull;
+		
+		// quickpath
+		if (!(*ptr & 0b1000'0000u)) CTRE_LIKELY {
+			return *ptr;
+		}
+ 
+		// calculate length based on first 5 bits
+		const unsigned length = (lengths >> ((*ptr >> 3) * 2)) & 0b11u;
+
+		// actual length is number + 1 bytes
+		
+		// length 0 here means it's a bad front unit
+		if (!length) CTRE_UNLIKELY {
+			return mojibake;
+		}
+
+		// if part of the utf-8 sequence is past the end
+		if (((ptr + length) >= end)) CTRE_UNLIKELY {
+			return mojibake;
+		}
+		
+		if ((ptr[1] >> 6) != 0b10) CTRE_UNLIKELY {
+			return mojibake;
+		}
+
+		const char8_t mask = (0b0010'0000u >> length) - 1;
+		
+		// length = 1 (2 bytes) mask = 0b0001'1111u
+		// length = 2 (3 bytes) mask = 0b0000'1111u
+		// length = 3 (4 bytes) mask = 0b0000'0111u
+
+		// remove utf8 front bits, get only significant part
+		// and add first trailing unit
+
+		char32_t result = ((ptr[0] & mask) << 6) | (ptr[1] & 0b0011'1111u);
+
+		// add rest of trailing units
+		if (length == 1) CTRE_LIKELY {
+			return result;
+		}
+
+		if ((ptr[2] >> 6) != 0b10) CTRE_UNLIKELY {
+			return mojibake;
+		}
+
+		result = (result << 6) | (ptr[2] & 0b0011'1111u);
+
+		if (length == 2) CTRE_LIKELY {
+			return result;
+		}
+
+		if ((ptr[3] >> 6) != 0b10) CTRE_UNLIKELY {
+			return mojibake;
+		}
+
+		return (result << 6) | (ptr[3] & 0b0011'1111u);
+	}
+};
+
+struct utf8_range {
+	std::u8string_view range;
+	constexpr utf8_range(std::u8string_view r) noexcept: range{r} { }
+	
+	constexpr auto begin() const noexcept {
+		return utf8_iterator{range.data(), range.data() + range.size()};
+	}
+	constexpr auto end() const noexcept {
+		return utf8_iterator::sentinel{};
+	}
+};
+
+}
+
+#endif
+
+#endif

include/ctre/utility.hpp

@@ -1,6 +1,18 @@
 #ifndef CTRE__UTILITY__HPP
 #define CTRE__UTILITY__HPP
 
+#if __has_cpp_attribute(likely)
+#define CTRE_LIKELY [[likely]]
+#else
+#define CTRE_LIKELY
+#endif
+
+#if __has_cpp_attribute(unlikely)
+#define CTRE_UNLIKELY [[unlikely]]
+#else
+#define CTRE_UNLIKELY
+#endif
+
 #ifdef _MSC_VER
 #define CTRE_FORCE_INLINE __forceinline
 #define CTRE_FLATTEN