Skip to content

CLDR-17599 Path descriptions: enable macro substitution #5055

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

CLDR-17599 Path descriptions: enable macro substitution #5055

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion tools/cldr-apps/src/test/java/org/unicode/cldr/util/TestCLDRLinks.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ public static Stream<Arguments> pathDescriptionProvider() {
// this gets all URLs in references
PathDescriptionParser pathDescriptionParser = new PathDescriptionParser();
String fileName = PathDescription.pathDescriptionFileName;
pathDescriptionParser.parse(PathDescription.getBigString(fileName));
pathDescriptionParser.parse(fileName);
for (final String url : urlsFromString(pathDescriptionParser.getReferences())) {
urls.putIfAbsent(url, fileName);
}
Expand Down
27 changes: 18 additions & 9 deletions tools/cldr-code/src/main/java/org/unicode/cldr/util/PathDescription.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,11 +58,9 @@ public enum ErrorHandling {

private static final PathDescriptionParser hintsParser = new PathDescriptionParser();

private static final RegexLookup<Pair<String, String>> pathHandling =
parser.parse(pathDescriptionString);
private static RegexLookup<Pair<String, String>> pathHandling = null;

private static final RegexLookup<Pair<String, String>> pathHintsHandling =
hintsParser.parse(pathDescriptionHintsString);
private static RegexLookup<Pair<String, String>> pathHintsHandling = null;

/** markdown to append */
private static final String references = parser.getReferences();
Expand All @@ -80,10 +78,20 @@ static String getBigString(String fileName) {
}

/** for tests */
static RegexLookup<Pair<String, String>> getPathHandling() {
public static RegexLookup<Pair<String, String>> getPathHandling() {
if (pathHandling == null) {
pathHandling = parser.parse(pathDescriptionFileName);
}
return pathHandling;
}

private static RegexLookup<Pair<String, String>> getPathHintsHandling() {
if (pathHintsHandling == null) {
pathHintsHandling = hintsParser.parse(pathDescriptionHintsFileName);
}
return pathHintsHandling;
}

// set in construction

private final CLDRFile english;
Expand Down Expand Up @@ -127,16 +135,17 @@ public enum Status {

public String getRawDescription(String path, Object context) {
status.clear();
final Pair<String, String> entry = pathHandling.get(path, context, pathArguments);
final Pair<String, String> entry = getPathHandling().get(path, context, pathArguments);
if (entry == null) {
return null;
}
return entry.getSecond();
}

public String getHintRawDescription(String path, Object context) {

status.clear();
final Pair<String, String> entry = pathHintsHandling.get(path, context, pathArguments);
final Pair<String, String> entry = getPathHintsHandling().get(path, context, pathArguments);
if (entry == null) {
return null;
}
Expand All @@ -147,7 +156,7 @@ public String getRawDescription(
String path, Object context, Output<Finder> matcherFound, Set<String> failures) {
status.clear();
final Pair<String, String> entry =
pathHandling.get(path, context, pathArguments, matcherFound, failures);
getPathHandling().get(path, context, pathArguments, matcherFound, failures);
if (entry == null) {
return null;
}
Expand All @@ -157,7 +166,7 @@ public String getRawDescription(
public String getDescription(String path, String value, Object context) {
status.clear();

final Pair<String, String> entry = pathHandling.get(path, context, pathArguments);
final Pair<String, String> entry = getPathHandling().get(path, context, pathArguments);
String description;
String markdown;
if (entry == null) {
Expand Down
77 changes: 50 additions & 27 deletions tools/cldr-code/src/main/java/org/unicode/cldr/util/PathDescriptionParser.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,19 +7,21 @@

public class PathDescriptionParser {
private static final String PROLOGUE = "PROLOGUE";
private static final String VARIABLES = "VARIABLES";
private static final String REFERENCES = "References";
final RegexLookup<Pair<String, String>> lookup =
private final RegexLookup<Pair<String, String>> lookup =
new RegexLookup<>(RegexLookup.LookupType.OPTIMIZED_DIRECTORY_PATTERN_LOOKUP);

// running instance variables
String section = ""; // ## title
String description = null; // ### title
List<String> patterns = new ArrayList<>();
List<String> text = new ArrayList<>();
List<String> referenceLines = new ArrayList<>();

public RegexLookup<Pair<String, String>> parse(String string) {
final String lines[] = string.split("\n");
private String section = ""; // ## title
private String description = null; // ### title
private final List<String> patterns = new ArrayList<>();
private final List<String> text = new ArrayList<>();
private final List<String> referenceLines = new ArrayList<>();

public RegexLookup<Pair<String, String>> parse(String fileName) {
String string = PathDescription.getBigString(fileName);
final String[] lines = string.split("\n");
int n = 0;

try {
Expand All @@ -30,34 +32,35 @@ public RegexLookup<Pair<String, String>> parse(String string) {
end(); // process last lines
} catch (Throwable t) {
throw new RuntimeException(
"Failed parsing PathDescriptions.md:"
+ n
+ ": at "
+ section
+ "#"
+ description,
"Failed parsing " + fileName + ":" + n + ": at " + section + "#" + description,
t);
}

return lookup;
}

/**
* @returns true if we're in the top section
* @return true if we're in the top section
*/
boolean inPrologue() {
private boolean inPrologue() {
return (section.equals(PROLOGUE));
}

boolean inReferences() {
private boolean inVariables() {
return (section.equals(VARIABLES));
}

private boolean inReferences() {
return (section.equals(REFERENCES));
}

void processLine(final String line) {
private void processLine(final String line) {
if (line.startsWith("#")) {
processHeader(line);
} else if (inPrologue() || beforePrologue()) {
return; // skip all other lines until new header
} else if (inVariables()) {
processVariable(line);
} else if (inReferences()) {
processReference(line);
} else if (line.startsWith("- `")) {
Expand All @@ -70,6 +73,19 @@ void processLine(final String line) {
}
}

private void processVariable(String line) {
if (line.isBlank()) {
return;
}
int pos = line.indexOf("=");
if (pos < 0) {
throw new IllegalArgumentException("Failed to read variable in " + line);
}
String varName = line.substring(0, pos).trim();
String varValue = line.substring(pos + 1).trim();
lookup.addVariable(varName, varValue);
}

private void processBody(String line) {
text.add(line);
}
Expand All @@ -88,7 +104,8 @@ public String getReferences() {
return String.join("\n", referenceLines).trim();
}

final Pattern HEADER_PATTERN = Pattern.compile("^(#+)(?:[ ]+(.*))?$");
private final Pattern HEADER_PATTERN = Pattern.compile("^(#+)(?:[ ]+(.*))?$");
private final Pattern VARIABLES_PATTERN = Pattern.compile("^# *VARIABLES *$");

private void processHeader(String line) {
endHeader(); // process previous content and reset
Expand All @@ -104,10 +121,17 @@ private void processHeader(String line) {
if (title == null) title = "";
title = title.trim();
if (headerLevel == 1) {
if (!beforePrologue()) {
throw new IllegalArgumentException("Extra # header after beginning of file");
if (VARIABLES_PATTERN.matcher(line).matches()) {
if (beforePrologue()) {
throw new IllegalArgumentException("Prologue should precede # " + VARIABLES);
}
section = VARIABLES;
} else {
if (!beforePrologue()) {
throw new IllegalArgumentException("Extra # header after beginning of file");
}
section = PROLOGUE;
}
section = PROLOGUE;
description = null;
} else if (headerLevel == 2) { // ##
if (beforePrologue()) {
Expand Down Expand Up @@ -149,7 +173,7 @@ private boolean beforePrologue() {
return section.isEmpty();
}

void processContent() {
private void processContent() {
final String textStr = String.join("\n", text).trim();
if (description == null) {
// have not had a ### section yet.
Expand All @@ -168,11 +192,10 @@ void processContent() {
} else if (patterns.size() > 1) {
throw new IllegalArgumentException("Only one pattern is supported at present.");
}

lookup.addWithoutVariables(patterns.get(0), Pair.of(description, textStr));
lookup.add(patterns.get(0), Pair.of(description, textStr));
}

void resetContent() {
private void resetContent() {
patterns.clear();
text.clear();
}
Expand Down
Loading
Loading