Inflection-63 Integrate ko Wikidata into Unicode Inflection

Inflection-62 Integrate ar Wikidata into Unicode Inflection
Inflection-61 Integrate he Wikidata into Unicode Inflection
Inflection-60 Integrate hi Wikidata into Unicode Inflection
Inflection-58 Integrate nb Wikidata into Unicode Inflection
Inflection-56 Integrate nl Wikidata into Unicode Inflection
Inflection-55 Integrate tr Wikidata into Unicode Inflection
Inflection-54 Integrate ru Wikidata into Unicode Inflection
Inflection-53 Integrate it Wikidata into Unicode Inflection
Inflection-52 Integrate pt Wikidata into Unicode Inflection
Inflection-51 Integrate fr Wikidata into Unicode Inflection
Inflection-50 Integrate de Wikidata into Unicode Inflection

Author: grhoten

.gitignore

@@ -1,5 +1,6 @@
-build/*
+.DS_Store
+.idea
 .vscode/launch.json
 .vscode/settings.json
+build
 fst/__pycache__/*
-inflection/tools/dictionary-parser/bin/*

inflection/cmake/versions.mk

@@ -1,7 +1,6 @@
 #
 # Copyright 2021-2024 Apple Inc. All rights reserved.
 #
-CATCH2_VERSION := 3.8.0
-CF_VERSION := 5.10.1
+CATCH2_VERSION := 3.9.0
 ICU_VERSION_MINIMUM := 77.1
-MARISA_VERSION := 0.2.6
+MARISA_VERSION := 0.3.1

inflection/ext/lib/Marisa/CMakeLists.txt

@@ -27,9 +27,7 @@ else()
     set(ARTIFACT "v${MARISA_VERSION}.tar.gz")
     ExternalProject_Add(marisa-trie-download
             URL https://github.com/s-yata/marisa-trie/archive/refs/tags/v${MARISA_VERSION}.tar.gz
-            # This patch is needed to address weak vtables, which is an issue when using full link time optimization.
-            # The patch may need to be updated if another version is created or additional patches are needed.
-            BUILD_COMMAND ${CMAKE_CURRENT_LIST_DIR}/patch.sh ${CMAKE_CURRENT_BINARY_DIR}/marisa-trie ${CMAKE_CURRENT_LIST_DIR}/Exception.patch
+            BUILD_COMMAND ""
             CONFIGURE_COMMAND "" INSTALL_COMMAND "" LOG_DOWNLOAD ON EXCLUDE_FROM_ALL TRUE
             SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}/marisa-trie
     )

inflection/ext/lib/Marisa/Exception.patch

@@ -12,6 +12,7 @@ Moyen-Orient: singular masculine noun proper-noun
 Tour Eiffel: singular feminine noun
 Vendée Globe: singular masculine noun
 hôtel: singular masculine noun vowel-start
+hôpital: singular masculine vowel-start noun
 ==============================================
 Manually curated for tests to pass
 Copyright 2024-2024 Apple Inc. All rights reserved.

inflection/ext/lib/Marisa/patch.sh

@@ -591,10 +591,10 @@
                 <restrictions>
                     <restriction name="pos" value="pronoun"/>
                 </restrictions>
-                <grammeme name="common"/> <!-- unspecified/unknown/epicene e.g. "them" or "one" -->
                 <grammeme name="masculine"/> <!-- e.g. "him" -->
                 <grammeme name="feminine"/> <!-- e.g. "her" -->
                 <grammeme name="neuter"/> <!-- e.g. "it", and not to be confused with "them" -->
+                <!-- null means unknown/epicene e.g. "them" or "one" -->
             </category>
             <category name="pronounType">
                 <restrictions>

inflection/resources/org/unicode/inflection/dictionary/supplemental_fr.lst

@@ -40083,7 +40083,7 @@ alegre	000000b9
 tigre	01000040
 ogre	0100002d
 ihre	00000031
-aire	01000054
+aire	0d000054
 hypothecaire	000000f6
 precaire	00000053
 bancaire	0000006a
@@ -120654,7 +120654,7 @@ partijcongres	00000142
 vrouwencongres	00000028
 vredescongres	0000002c
 volkscongres	0000004e
-aires	010002f5
+aires	0d000055
 affaires	0000009c
 claires	01000036
 baudelaires	0000002c

inflection/resources/org/unicode/inflection/features/grammar.xml

@@ -23,6 +23,7 @@ RussianExposableMorphology::RussianExposableMorphology()
         {GrammemeConstants::GENDER_MASCULINE(), GrammemeConstants::GENDER_FEMININE(), GrammemeConstants::GENDER_NEUTER()},
         {GrammemeConstants::NUMBER_SINGULAR(), GrammemeConstants::NUMBER_PLURAL()},
         {GrammemeConstants::ANIMACY_INANIMATE(), GrammemeConstants::ANIMACY_ANIMATE()},
+        {GrammemeConstants::REGISTER_INFORMAL()}
     }, {{GrammemeConstants::POS_PROPER_NOUN(), GrammemeConstants::NUMBER_PLURAL()}}, true)
 {
     const auto &dictionary = getDictionary();

inflection/resources/org/unicode/inflection/tokenizer/nl/tokenizer.dictionary

@@ -12,7 +12,7 @@
 
 namespace inflection::dialog {
 
-static bool getFeatureValues(::std::set<::std::u16string>* articles, const std::vector<::inflection::lang::features::LanguageGrammarFeatures_Feature>& featureValues, const ::std::set<::std::u16string>& excludeValues)
+static bool getFeatureValues(::std::set<::std::u16string, std::less<>>* articles, const std::vector<::inflection::lang::features::LanguageGrammarFeatures_Feature>& featureValues, const ::std::set<::std::u16string, std::less<>>& excludeValues)
 {
     bool hasTrailingSpace = false;
     bool normalizeApostrophe = false;
@@ -28,7 +28,7 @@ static bool getFeatureValues(::std::set<::std::u16string>* articles, const std::
     }
     for (const auto& featureValue : featureValues) {
         std::u16string featureValueStr(featureValue.getValue());
-        if (featureValueStr.empty() || excludeValues.find(featureValueStr) != excludeValues.end()) {
+        if (featureValueStr.empty() || excludeValues.contains(featureValueStr)) {
             continue;
         }
         normalizeApostrophe = normalizeApostrophe || featureValueStr.find(u'’') != std::u16string::npos;
@@ -37,7 +37,7 @@ static bool getFeatureValues(::std::set<::std::u16string>* articles, const std::
     return normalizeApostrophe;
 }
 
-ArticleDetectionFunction::ArticleDetectionFunction(const ::inflection::util::ULocale& locale, const ::std::set<::std::u16string>& definiteFeatures, const ::std::set<::std::u16string>& excludeDefiniteValues, const ::std::set<::std::u16string>& indefiniteFeatures, const ::std::set<::std::u16string>& excludeIndefiniteValues)
+ArticleDetectionFunction::ArticleDetectionFunction(const ::inflection::util::ULocale& locale, const ::std::set<::std::u16string, std::less<>>& definiteFeatures, const ::std::set<::std::u16string, std::less<>>& excludeDefiniteValues, const ::std::set<::std::u16string, std::less<>>& indefiniteFeatures, const ::std::set<::std::u16string, std::less<>>& excludeIndefiniteValues)
     : super()
     , locale(locale)
 {
@@ -47,10 +47,10 @@ ArticleDetectionFunction::ArticleDetectionFunction(const ::inflection::util::ULo
         if (values.empty()) {
             continue;
         }
-        if (definiteFeatures.find(feature.getName()) != definiteFeatures.end()) {
+        if (definiteFeatures.contains(feature.getName())) {
             normalizeApostrophe = getFeatureValues(&definiteArticles, values, excludeDefiniteValues) || normalizeApostrophe;
         }
-        if (indefiniteFeatures.find(feature.getName()) != indefiniteFeatures.end()) {
+        if (indefiniteFeatures.contains(feature.getName())) {
             normalizeApostrophe = getFeatureValues(&indefiniteArticles, values, excludeIndefiniteValues) || normalizeApostrophe;
         }
     }