Skip to content

CLDR-17014 Move more extra-path code from CLDRFile to ExtraPaths #4332

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
btangmu merged 2 commits into from
Feb 3, 2025
+281 −286
Merged

CLDR-17014 Move more extra-path code from CLDRFile to ExtraPaths #4332

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
345d65e
CLDR-17014 Move more extra-path code from CLDRFile to ExtraPaths
btangmu Feb 3, 2025
b3f6da3
CLDR-17014 Move more extra-path code from CLDRFile to ExtraPaths
btangmu Feb 3, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
289 changes: 4 additions & 285 deletions tools/cldr-code/src/main/java/org/unicode/cldr/util/CLDRFile.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,6 @@
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
Expand All @@ -53,15 +52,9 @@
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import org.unicode.cldr.test.CheckMetazones;
import org.unicode.cldr.util.DayPeriodInfo.DayPeriod;
import org.unicode.cldr.util.GrammarInfo.GrammaticalFeature;
import org.unicode.cldr.util.GrammarInfo.GrammaticalScope;
import org.unicode.cldr.util.GrammarInfo.GrammaticalTarget;
import org.unicode.cldr.util.LocaleInheritanceInfo.Reason;
import org.unicode.cldr.util.SupplementalDataInfo.PluralInfo;
import org.unicode.cldr.util.SupplementalDataInfo.PluralInfo.Count;
import org.unicode.cldr.util.SupplementalDataInfo.PluralType;
import org.unicode.cldr.util.With.SimpleIterator;
import org.unicode.cldr.util.XMLFileReader.AllHandler;
import org.unicode.cldr.util.XMLSource.ResolvingSource;
Expand Down Expand Up @@ -98,9 +91,6 @@

public class CLDRFile implements Freezable<CLDRFile>, Iterable<String>, LocaleStringProvider {

private static final ImmutableSet<String> casesNominativeOnly =
ImmutableSet.of(GrammaticalFeature.grammaticalCase.getDefault(null));

/**
* Variable to control whether File reads are buffered; this will about halve the time spent in
* loadFromFile() and Factory.make() from about 20 % to about 10 %. It will also noticeably
Expand Down Expand Up @@ -3082,8 +3072,7 @@ public String getWinningPath(String path) {
* @return
*/
public Collection<String> getExtraPaths() {
Set<String> toAddTo = new HashSet<>();
toAddTo.addAll(getRawExtraPaths());
Set<String> toAddTo = new HashSet<>(getRawExtraPaths());
for (String path : this) {
toAddTo.remove(path);
}
Expand Down Expand Up @@ -3116,11 +3105,7 @@ public Collection<String> getExtraPaths(String prefix, Collection<String> toAddT
*/
public Set<String> getRawExtraPaths() {
if (extraPaths == null) {
extraPaths =
ImmutableSet.<String>builder()
.addAll(getRawExtraPathsPrivate())
.addAll(CONST_EXTRA_PATHS)
.build();
extraPaths = ImmutableSet.<String>builder().addAll(getRawExtraPathsPrivate()).build();
if (DEBUG) {
System.out.println(getLocaleID() + "\textras: " + extraPaths.size());
}
Expand All @@ -3134,8 +3119,6 @@ public Set<String> getRawExtraPaths() {
* are common to all locales, and some of them are specific to the given locale, based on
* features like the plural rules for the locale.
*
* <p>The ones that are constant for all locales should go into CONST_EXTRA_PATHS.
*
* @return toAddTo (the collection)
* <p>Called only by getRawExtraPaths.
* <p>"Raw" refers to the fact that some of the paths may duplicate paths that are already
Expand All @@ -3150,227 +3133,11 @@ public Set<String> getRawExtraPaths() {
*/
private List<String> getRawExtraPathsPrivate() {
Set<String> toAddTo = new HashSet<>();
SupplementalDataInfo supplementalData = CLDRConfig.getInstance().getSupplementalDataInfo();

ExtraPaths.add(toAddTo);

// units
PluralInfo plurals = supplementalData.getPlurals(PluralType.cardinal, getLocaleID());
if (plurals == null && DEBUG) {
System.err.println(
"No "
+ PluralType.cardinal
+ " plurals for "
+ getLocaleID()
+ " in "
+ supplementalData.getDirectory().getAbsolutePath());
}
Set<Count> pluralCounts = Collections.emptySet();
if (plurals != null) {
pluralCounts = plurals.getAdjustedCounts();
Set<Count> pluralCountsRaw = plurals.getCounts();
if (pluralCountsRaw.size() != 1) {
// we get all the root paths with count
addPluralCounts(toAddTo, pluralCounts, pluralCountsRaw, this);
}
}
// dayPeriods
String locale = getLocaleID();
DayPeriodInfo dayPeriods =
supplementalData.getDayPeriods(DayPeriodInfo.Type.format, locale);
if (dayPeriods != null) {
LinkedHashSet<DayPeriod> items = new LinkedHashSet<>(dayPeriods.getPeriods());
items.add(DayPeriod.am);
items.add(DayPeriod.pm);
for (String context : new String[] {"format", "stand-alone"}) {
for (String width : new String[] {"narrow", "abbreviated", "wide"}) {
for (DayPeriod dayPeriod : items) {
// ldml/dates/calendars/calendar[@type="gregorian"]/dayPeriods/dayPeriodContext[@type="format"]/dayPeriodWidth[@type="wide"]/dayPeriod[@type="am"]
toAddTo.add(
"//ldml/dates/calendars/calendar[@type=\"gregorian\"]/dayPeriods/"
+ "dayPeriodContext[@type=\""
+ context
+ "\"]/dayPeriodWidth[@type=\""
+ width
+ "\"]/dayPeriod[@type=\""
+ dayPeriod
+ "\"]");
}
}
}
}

// metazones
Set<String> zones = supplementalData.getAllMetazones();

for (String zone : zones) {
final boolean metazoneUsesDST = CheckMetazones.metazoneUsesDST(zone);
for (String width : new String[] {"long", "short"}) {
for (String type : new String[] {"generic", "standard", "daylight"}) {
if (metazoneUsesDST || type.equals("standard")) {
// Only add /standard for non-DST metazones
final String path =
"//ldml/dates/timeZoneNames/metazone[@type=\""
+ zone
+ "\"]/"
+ width
+ "/"
+ type;
toAddTo.add(path);
}
}
}
}

// // Individual zone overrides
// final String[] overrides = {
// "Pacific/Honolulu\"]/short/generic",
// "Pacific/Honolulu\"]/short/standard",
// "Pacific/Honolulu\"]/short/daylight",
// "Europe/Dublin\"]/long/daylight",
// "Europe/London\"]/long/daylight",
// "Etc/UTC\"]/long/standard",
// "Etc/UTC\"]/short/standard"
// };
// for (String override : overrides) {
// toAddTo.add("//ldml/dates/timeZoneNames/zone[@type=\"" + override);
// }

// Currencies
Set<String> codes = supplementalData.getBcp47Keys().getAll("cu");
for (String code : codes) {
String currencyCode = code.toUpperCase();
toAddTo.add(
"//ldml/numbers/currencies/currency[@type=\"" + currencyCode + "\"]/symbol");
toAddTo.add(
"//ldml/numbers/currencies/currency[@type=\""
+ currencyCode
+ "\"]/displayName");
if (!pluralCounts.isEmpty()) {
for (Count count : pluralCounts) {
toAddTo.add(
"//ldml/numbers/currencies/currency[@type=\""
+ currencyCode
+ "\"]/displayName[@count=\""
+ count.toString()
+ "\"]");
}
}
}

// grammatical info

GrammarInfo grammarInfo = supplementalData.getGrammarInfo(getLocaleID(), true);
if (grammarInfo != null) {
if (grammarInfo.hasInfo(GrammaticalTarget.nominal)) {
Collection<String> genders =
grammarInfo.get(
GrammaticalTarget.nominal,
GrammaticalFeature.grammaticalGender,
GrammaticalScope.units);
Collection<String> rawCases =
grammarInfo.get(
GrammaticalTarget.nominal,
GrammaticalFeature.grammaticalCase,
GrammaticalScope.units);
Collection<String> nomCases = rawCases.isEmpty() ? casesNominativeOnly : rawCases;
Collection<Count> adjustedPlurals = pluralCounts;
// There was code here allowing fewer plurals to be used, but is retracted for now
// (needs more thorough integration in logical groups, etc.)
// This note is left for 'blame' to find the old code in case we revive that.

// TODO use UnitPathType to get paths
if (!genders.isEmpty()) {
for (String unit : GrammarInfo.getUnitsToAddGrammar()) {
toAddTo.add(
"//ldml/units/unitLength[@type=\"long\"]/unit[@type=\""
+ unit
+ "\"]/gender");
}
for (Count plural : adjustedPlurals) {
for (String gender : genders) {
for (String case1 : nomCases) {
final String grammaticalAttributes =
GrammarInfo.getGrammaticalInfoAttributes(
grammarInfo,
UnitPathType.power,
plural.toString(),
gender,
case1);
toAddTo.add(
"//ldml/units/unitLength[@type=\"long\"]/compoundUnit[@type=\"power2\"]/compoundUnitPattern1"
+ grammaticalAttributes);
toAddTo.add(
"//ldml/units/unitLength[@type=\"long\"]/compoundUnit[@type=\"power3\"]/compoundUnitPattern1"
+ grammaticalAttributes);
}
}
}
// <genderMinimalPairs gender="masculine">Der {0} ist
// …</genderMinimalPairs>
for (String gender : genders) {
toAddTo.add(
"//ldml/numbers/minimalPairs/genderMinimalPairs[@gender=\""
+ gender
+ "\"]");
}
}
if (!rawCases.isEmpty()) {
for (String case1 : rawCases) {
// <caseMinimalPairs case="nominative">{0} kostet
// €3,50.</caseMinimalPairs>
toAddTo.add(
"//ldml/numbers/minimalPairs/caseMinimalPairs[@case=\""
+ case1
+ "\"]");

for (Count plural : adjustedPlurals) {
for (String unit : GrammarInfo.getUnitsToAddGrammar()) {
toAddTo.add(
"//ldml/units/unitLength[@type=\"long\"]/unit[@type=\""
+ unit
+ "\"]/unitPattern"
+ GrammarInfo.getGrammaticalInfoAttributes(
grammarInfo,
UnitPathType.unit,
plural.toString(),
null,
case1));
}
}
}
}
}
}
ExtraPaths.addConstant(toAddTo);
ExtraPaths.addLocaleDependent(toAddTo, this, getLocaleID());
return toAddTo.stream().map(String::intern).collect(Collectors.toList());
}

private void addPluralCounts(
Collection<String> toAddTo,
final Set<Count> pluralCounts,
final Set<Count> pluralCountsRaw,
Iterable<String> file) {
for (String path : file) {
String countAttr = "[@count=\"other\"]";
int countPos = path.indexOf(countAttr);
if (countPos < 0) {
continue;
}
Set<Count> pluralCountsNeeded =
path.startsWith("//ldml/numbers/minimalPairs") ? pluralCountsRaw : pluralCounts;
if (pluralCountsNeeded.size() > 1) {
String start = path.substring(0, countPos) + "[@count=\"";
String end = "\"]" + path.substring(countPos + countAttr.length());
for (Count count : pluralCounts) {
if (count == Count.other) {
continue;
}
toAddTo.add(start + count + end);
}
}
}
}

/**
* Get the path with the given count, case, or gender, with fallback. The fallback acts like an
* alias in root.
Expand Down Expand Up @@ -3744,52 +3511,4 @@ String getStringValueWithBaileyNotConstructed(String path) {
}
return value;
}

/**
* A set of paths to be added to getRawExtraPaths(). These are constant across locales, and
* don't have good fallback values in root. NOTE: if this is changed, you'll need to modify
* TestPaths.extraPathAllowsNullValue
*/
static final Set<String> CONST_EXTRA_PATHS =
CharUtilities.internImmutableSet(
Set.of(
// Individual zone overrides — were in getRawExtraPaths
"//ldml/dates/timeZoneNames/zone[@type=\"Pacific/Honolulu\"]/short/generic",
"//ldml/dates/timeZoneNames/zone[@type=\"Pacific/Honolulu\"]/short/standard",
"//ldml/dates/timeZoneNames/zone[@type=\"Pacific/Honolulu\"]/short/daylight",
"//ldml/dates/timeZoneNames/zone[@type=\"Europe/Dublin\"]/long/daylight",
"//ldml/dates/timeZoneNames/zone[@type=\"Europe/London\"]/long/daylight",
"//ldml/dates/timeZoneNames/zone[@type=\"Etc/UTC\"]/long/standard",
"//ldml/dates/timeZoneNames/zone[@type=\"Etc/UTC\"]/short/standard",
// Person name paths
"//ldml/personNames/sampleName[@item=\"nativeG\"]/nameField[@type=\"given\"]",
"//ldml/personNames/sampleName[@item=\"nativeGS\"]/nameField[@type=\"given\"]",
"//ldml/personNames/sampleName[@item=\"nativeGS\"]/nameField[@type=\"surname\"]",
"//ldml/personNames/sampleName[@item=\"nativeGGS\"]/nameField[@type=\"given\"]",
"//ldml/personNames/sampleName[@item=\"nativeGGS\"]/nameField[@type=\"given2\"]",
"//ldml/personNames/sampleName[@item=\"nativeGGS\"]/nameField[@type=\"surname\"]",
"//ldml/personNames/sampleName[@item=\"nativeFull\"]/nameField[@type=\"title\"]",
"//ldml/personNames/sampleName[@item=\"nativeFull\"]/nameField[@type=\"given\"]",
"//ldml/personNames/sampleName[@item=\"nativeFull\"]/nameField[@type=\"given-informal\"]",
"//ldml/personNames/sampleName[@item=\"nativeFull\"]/nameField[@type=\"given2\"]",
"//ldml/personNames/sampleName[@item=\"nativeFull\"]/nameField[@type=\"surname-prefix\"]",
"//ldml/personNames/sampleName[@item=\"nativeFull\"]/nameField[@type=\"surname-core\"]",
"//ldml/personNames/sampleName[@item=\"nativeFull\"]/nameField[@type=\"surname2\"]",
"//ldml/personNames/sampleName[@item=\"nativeFull\"]/nameField[@type=\"generation\"]",
"//ldml/personNames/sampleName[@item=\"nativeFull\"]/nameField[@type=\"credentials\"]",
"//ldml/personNames/sampleName[@item=\"foreignG\"]/nameField[@type=\"given\"]",
"//ldml/personNames/sampleName[@item=\"foreignGS\"]/nameField[@type=\"given\"]",
"//ldml/personNames/sampleName[@item=\"foreignGS\"]/nameField[@type=\"surname\"]",
"//ldml/personNames/sampleName[@item=\"foreignGGS\"]/nameField[@type=\"given\"]",
"//ldml/personNames/sampleName[@item=\"foreignGGS\"]/nameField[@type=\"given2\"]",
"//ldml/personNames/sampleName[@item=\"foreignGGS\"]/nameField[@type=\"surname\"]",
"//ldml/personNames/sampleName[@item=\"foreignFull\"]/nameField[@type=\"title\"]",
"//ldml/personNames/sampleName[@item=\"foreignFull\"]/nameField[@type=\"given\"]",
"//ldml/personNames/sampleName[@item=\"foreignFull\"]/nameField[@type=\"given-informal\"]",
"//ldml/personNames/sampleName[@item=\"foreignFull\"]/nameField[@type=\"given2\"]",
"//ldml/personNames/sampleName[@item=\"foreignFull\"]/nameField[@type=\"surname-prefix\"]",
"//ldml/personNames/sampleName[@item=\"foreignFull\"]/nameField[@type=\"surname-core\"]",
"//ldml/personNames/sampleName[@item=\"foreignFull\"]/nameField[@type=\"surname2\"]",
"//ldml/personNames/sampleName[@item=\"foreignFull\"]/nameField[@type=\"generation\"]",
"//ldml/personNames/sampleName[@item=\"foreignFull\"]/nameField[@type=\"credentials\"]"));
}
Loading
Loading