Add is_valid_letter() to TextServer

MJacred · MJacred · commit 717513a62d97 · 2024-05-02T21:32:20.000+02:00
diff --git a/core/string/char_range.inc b/core/string/char_range.inc
diff --git a/core/string/char_utils.h b/core/string/char_utils.h
@@ -70,6 +70,10 @@ static _FORCE_INLINE_ bool is_unicode_lower_case(char32_t c) {
 	BSEARCH_CHAR_RANGE(lowercase_letter);
 }
 
+static _FORCE_INLINE_ bool is_unicode_letter(char32_t c) {
+	BSEARCH_CHAR_RANGE(unicode_letter);
+}
+
 #undef BSEARCH_CHAR_RANGE
 
 static _FORCE_INLINE_ bool is_ascii_upper_case(char32_t c) {
diff --git a/doc/classes/TextServer.xml b/doc/classes/TextServer.xml
@@ -1088,6 +1088,13 @@
 				- May contain Unicode characters of class XID_Continue in the other positions.
 			</description>
 		</method>
+		<method name="is_valid_letter" qualifiers="const">
+			<return type="bool" />
+			<param index="0" name="unicode" type="int" />
+			<description>
+				Returns [code]true[/code] if the given code point is a valid letter, i.e. it belongs to the Unicode category "L".
+			</description>
+		</method>
 		<method name="load_support_data">
 			<return type="bool" />
 			<param index="0" name="filename" type="String" />
diff --git a/doc/classes/TextServerExtension.xml b/doc/classes/TextServerExtension.xml
@@ -937,6 +937,12 @@
 			<description>
 			</description>
 		</method>
+		<method name="_is_valid_letter" qualifiers="virtual const">
+			<return type="bool" />
+			<param index="0" name="unicode" type="int" />
+			<description>
+			</description>
+		</method>
 		<method name="_load_support_data" qualifiers="virtual">
 			<return type="bool" />
 			<param index="0" name="filename" type="String" />
diff --git a/modules/text_server_adv/text_server_adv.cpp b/modules/text_server_adv/text_server_adv.cpp
@@ -7229,6 +7229,16 @@ bool TextServerAdvanced::_is_valid_identifier(const String &p_string) const {
 	return true;
 }
 
+bool TextServerAdvanced::_is_valid_letter(char32_t p_unicode) const {
+#ifndef ICU_STATIC_DATA
+	if (!icu_data_loaded) {
+		return TextServer::is_valid_letter(p_unicode);
+	}
+#endif
+
+	return u_isalpha(p_unicode);
+}
+
 TextServerAdvanced::TextServerAdvanced() {
 	_insert_num_systems_lang();
 	_insert_feature_sets();
diff --git a/modules/text_server_adv/text_server_adv.h b/modules/text_server_adv/text_server_adv.h
@@ -979,6 +979,7 @@ class TextServerAdvanced : public TextServerExtension {
 
 	MODBIND1RC(String, strip_diacritics, const String &);
 	MODBIND1RC(bool, is_valid_identifier, const String &);
+	MODBIND1RC(bool, is_valid_letter, char32_t);
 
 	MODBIND2RC(String, string_to_upper, const String &, const String &);
 	MODBIND2RC(String, string_to_lower, const String &, const String &);
diff --git a/servers/text/text_server_extension.cpp b/servers/text/text_server_extension.cpp
@@ -327,6 +327,7 @@ void TextServerExtension::_bind_methods() {
 
 	GDVIRTUAL_BIND(_strip_diacritics, "string");
 	GDVIRTUAL_BIND(_is_valid_identifier, "string");
+	GDVIRTUAL_BIND(_is_valid_letter, "unicode");
 
 	GDVIRTUAL_BIND(_string_get_word_breaks, "string", "language", "chars_per_line");
 	GDVIRTUAL_BIND(_string_get_character_breaks, "string", "language");
@@ -1460,6 +1461,14 @@ bool TextServerExtension::is_valid_identifier(const String &p_string) const {
 	return TextServer::is_valid_identifier(p_string);
 }
 
+bool TextServerExtension::is_valid_letter(char32_t p_unicode) const {
+	bool ret;
+	if (GDVIRTUAL_CALL(_is_valid_letter, p_unicode, ret)) {
+		return ret;
+	}
+	return TextServer::is_valid_letter(p_unicode);
+}
+
 String TextServerExtension::strip_diacritics(const String &p_string) const {
 	String ret;
 	if (GDVIRTUAL_CALL(_strip_diacritics, p_string, ret)) {
diff --git a/servers/text/text_server_extension.h b/servers/text/text_server_extension.h
@@ -554,6 +554,8 @@ class TextServerExtension : public TextServer {
 
 	virtual bool is_valid_identifier(const String &p_string) const override;
 	GDVIRTUAL1RC(bool, _is_valid_identifier, const String &);
+	virtual bool is_valid_letter(char32_t p_unicode) const override;
+	GDVIRTUAL1RC(bool, _is_valid_letter, char32_t);
 
 	virtual String string_to_upper(const String &p_string, const String &p_language = "") const override;
 	virtual String string_to_lower(const String &p_string, const String &p_language = "") const override;
diff --git a/servers/text_server.cpp b/servers/text_server.cpp
@@ -483,6 +483,7 @@ void TextServer::_bind_methods() {
 
 	ClassDB::bind_method(D_METHOD("strip_diacritics", "string"), &TextServer::strip_diacritics);
 	ClassDB::bind_method(D_METHOD("is_valid_identifier", "string"), &TextServer::is_valid_identifier);
+	ClassDB::bind_method(D_METHOD("is_valid_letter", "unicode"), &TextServer::is_valid_letter);
 
 	ClassDB::bind_method(D_METHOD("string_to_upper", "string", "language"), &TextServer::string_to_upper, DEFVAL(""));
 	ClassDB::bind_method(D_METHOD("string_to_lower", "string", "language"), &TextServer::string_to_lower, DEFVAL(""));
@@ -2091,6 +2092,10 @@ bool TextServer::is_valid_identifier(const String &p_string) const {
 	return true;
 }
 
+bool TextServer::is_valid_letter(char32_t p_unicode) const {
+	return is_unicode_letter(p_unicode);
+}
+
 TextServer::TextServer() {
 	_init_diacritics_map();
 }
diff --git a/servers/text_server.h b/servers/text_server.h
@@ -534,6 +534,7 @@ class TextServer : public RefCounted {
 
 	virtual String strip_diacritics(const String &p_string) const;
 	virtual bool is_valid_identifier(const String &p_string) const;
+	virtual bool is_valid_letter(char32_t p_unicode) const;
 
 	// Other string operations.
 	virtual String string_to_upper(const String &p_string, const String &p_language = "") const = 0;
diff --git a/tests/servers/test_text_server.h b/tests/servers/test_text_server.h
@@ -637,6 +637,97 @@ TEST_SUITE("[TextServer]") {
 			}
 		}
 
+		SUBCASE("[TextServer] Unicode letters") {
+			for (int i = 0; i < TextServerManager::get_singleton()->get_interface_count(); i++) {
+				Ref<TextServer> ts = TextServerManager::get_singleton()->get_interface(i);
+				CHECK_FALSE_MESSAGE(ts.is_null(), "Invalid TS interface.");
+
+				struct ul_testcase {
+					int fail_index = -1; // Expecting failure at given index.
+					char32_t text[10]; // Using 0 as the terminator.
+				};
+				ul_testcase cases[14] = {
+					{
+							0,
+							{ 0x2D, 0x33, 0x30, 0, 0, 0, 0, 0, 0, 0 }, // "-30"
+					},
+					{
+							1,
+							{ 0x61, 0x2E, 0x31, 0, 0, 0, 0, 0, 0, 0 }, // "a.1"
+					},
+					{
+							1,
+							{ 0x61, 0x2C, 0x31, 0, 0, 0, 0, 0, 0, 0 }, // "a,1"
+					},
+					{
+							0,
+							{ 0x31, 0x65, 0x2D, 0x32, 0, 0, 0, 0, 0, 0 }, // "1e-2"
+					},
+					{
+							0,
+							{ 0xAB, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, // "Left-Pointing Double Angle Quotation Mark"
+					},
+					{
+							-1,
+							{ 0x41, 0x42, 0, 0, 0, 0, 0, 0, 0, 0 }, // "AB"
+					},
+					{
+							4,
+							{ 0x54, 0x65, 0x73, 0x74, 0x31, 0, 0, 0, 0, 0 }, // "Test1"
+					},
+					{
+							2,
+							{ 0x54, 0x65, 0x2A, 0x73, 0x74, 0, 0, 0, 0, 0 }, // "Te*st"
+					},
+					{
+							4,
+							{ 0x74, 0x65, 0x73, 0x74, 0x5F, 0x74, 0x65, 0x73, 0x74, 0x65 }, // "test_teste"
+					},
+					{
+							4,
+							{ 0x74, 0x65, 0x73, 0x74, 0x20, 0x74, 0x65, 0x73, 0x74, 0 }, // "test test"
+					},
+					{
+							-1,
+							{ 0x643, 0x402, 0x716, 0xB05, 0, 0, 0, 0, 0, 0 }, // "كЂܖଅ" (arabic letters),
+					},
+					{
+							-1,
+							{ 0x643, 0x402, 0x716, 0xB05, 0x54, 0x65, 0x73, 0x74, 0x30AA, 0x4E21 }, // 0-3 arabic letters, 4-7 latin letters, 8-9 CJK letters
+					},
+					{
+							-1,
+							{ 0x4D2, 0x4D6, 0x4DA, 0x4DC, 0, 0, 0, 0, 0, 0 }, // "ӒӖӚӜ" cyrillic letters
+					},
+					{
+							-1,
+							{ 0xC2, 0xC3, 0xC4, 0xC5, 0x100, 0x102, 0x104, 0xC7, 0x106, 0x108 }, // "ÂÃÄÅĀĂĄÇĆĈ" rarer latin letters
+					},
+				};
+
+				for (int j = 0; j < 14; j++) {
+					ul_testcase test = cases[j];
+					int failed_on_index = -1;
+					for (int k = 0; k < 10; k++) {
+						char32_t character = test.text[k];
+						if (character == 0) {
+							break;
+						}
+						if (!ts->is_valid_letter(character)) {
+							failed_on_index = k;
+							break;
+						}
+					}
+
+					if (test.fail_index == -1) {
+						CHECK_MESSAGE(test.fail_index == failed_on_index, "In interface ", ts->get_name() + ": In test case ", j, ", the character at index ", failed_on_index, " should have been a letter.");
+					} else {
+						CHECK_MESSAGE(test.fail_index == failed_on_index, "In interface ", ts->get_name() + ": In test case ", j, ", expected first non-letter at index ", test.fail_index, ", but found at index ", failed_on_index);
+					}
+				}
+			}
+		}
+
 		SUBCASE("[TextServer] Strip Diacritics") {
 			for (int i = 0; i < TextServerManager::get_singleton()->get_interface_count(); i++) {
 				Ref<TextServer> ts = TextServerManager::get_singleton()->get_interface(i);

Original file line number	Diff line number	Diff line change
`@@ -70,6 +70,10 @@ static _FORCE_INLINE_ bool is_unicode_lower_case(char32_t c) {`
`70`	`70`	`BSEARCH_CHAR_RANGE(lowercase_letter);`
`71`	`71`	`}`
`72`	`72`
	`73`	`+static _FORCE_INLINE_ bool is_unicode_letter(char32_t c) {`
	`74`	`+ BSEARCH_CHAR_RANGE(unicode_letter);`
	`75`	`+}`
	`76`	`+`
`73`	`77`	`#undef BSEARCH_CHAR_RANGE`
`74`	`78`
`75`	`79`	`static _FORCE_INLINE_ bool is_ascii_upper_case(char32_t c) {`