Skip to content

Commit 5cb43cb

Browse files
echeranecheran
authored
Clean up output dir settings (#173)
* Add CI workflow that runs post-Mavenization CLI commands as described by docs/build.md * Fix paths and other typos for out-of-source build instructions * Rename Java system property UNICODETOOLS_OUTPUT_DIR to UNICODETOOLS_GEN_DIR and update vals in docs and CI that preserves dir structure * Update Unicode version to 15.0.0 in new CI build workflow file * Remove the `output` segment from Generated file paths for out-of-source builds
1 parent 79c82d2 commit 5cb43cb

File tree

20 files changed

+147
-61
lines changed

20 files changed

+147
-61
lines changed

‎.github/workflows/build-jsp.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,13 +43,13 @@ jobs:
4343
-Dexec.mainClass="org.unicode.text.UCD.Main" -Dexec.args="version 15.0.0 build MakeUnicodeFiles"
4444
-pl unicodetools -DCLDR_DIR=${GITHUB_WORKSPACE}/cldr
4545
-DUNICODETOOLS_REPO_DIR=$(pwd) -DUVERSION=15.0.0
46-
-DUNICODETOOLS_OUTPUT_DIR=$(pwd)/output
46+
-DUNICODETOOLS_GEN_DIR=$(pwd)/output/Generated
4747
env:
4848
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
4949
- name: Run unicodetools tests
5050
run: >
5151
mvn -s .github/workflows/mvn-settings.xml -B test -pl unicodetools
52-
-DCLDR_DIR=${GITHUB_WORKSPACE}/cldr -T 1C -Dparallel=classes -DUNICODETOOLS_REPO_DIR=$(pwd) -DUVERSION=15.0.0 -DUNICODETOOLS_OUTPUT_DIR=$(pwd)/output
52+
-DCLDR_DIR=${GITHUB_WORKSPACE}/cldr -T 1C -Dparallel=classes -DUNICODETOOLS_REPO_DIR=$(pwd) -DUVERSION=15.0.0 -DUNICODETOOLS_GEN_DIR=$(pwd)/output/Generated
5353
env:
5454
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
5555
- name: Package JSPs

‎.github/workflows/cli-build-instructions.yml‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
name: build.md
2+
3+
on:
4+
push:
5+
branches:
6+
- '*'
7+
pull_request:
8+
branches:
9+
- '*'
10+
11+
jobs:
12+
in-source-build:
13+
name: In-source Instructions
14+
runs-on: ubuntu-latest
15+
steps:
16+
- name: Checkout Unicode Tools
17+
uses: actions/checkout@v2
18+
- name: Check out CLDR
19+
uses: actions/checkout@v2
20+
with:
21+
repository: unicode-org/cldr
22+
path: cldr
23+
- name: Move CLDR working copy to be sibling of Unicode Tools
24+
run: |
25+
mv cldr ..
26+
- name: Set up JDK 11
27+
uses: actions/setup-java@v1
28+
with:
29+
java-version: 11
30+
- name: Cache local Maven repository
31+
uses: actions/cache@v2
32+
with:
33+
path: ~/.m2/repository
34+
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
35+
restore-keys: |
36+
${{ runner.os }}-maven-
37+
- name: Set up in-source output dir
38+
run: |
39+
mkdir -p output/Generated/BIN
40+
- name: Initialize Maven settings for all Maven commands
41+
run: mvn -s .github/workflows/mvn-settings.xml -B compile install -DskipTests=true
42+
env:
43+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
44+
- name: Run command - Make Unicode Files
45+
run: MAVEN_OPTS="-ea" mvn -s .github/workflows/mvn-settings.xml exec:java -Dexec.mainClass="org.unicode.text.UCD.Main" -Dexec.args="version 15.0.0 build MakeUnicodeFiles" -pl unicodetools -DCLDR_DIR=$(cd ../cldr ; pwd) -DUNICODETOOLS_GEN_DIR=$(cd output/Generated; pwd) -DUNICODETOOLS_REPO_DIR=$(pwd) -DUVERSION=15.0.0
46+
env:
47+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
48+
- name: Run command - Build and Test
49+
run: MAVEN_OPTS="-ea" mvn -s .github/workflows/mvn-settings.xml package -DCLDR_DIR=$(cd ../cldr ; pwd) -DUNICODETOOLS_GEN_DIR=$(cd output/Generated; pwd) -DUNICODETOOLS_REPO_DIR=$(pwd) -DUVERSION=15.0.0
50+
env:
51+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
52+
53+
out-of-source-build:
54+
name: Out-of-source Instructions
55+
runs-on: ubuntu-latest
56+
steps:
57+
- name: Checkout Unicode Tools
58+
uses: actions/checkout@v2
59+
with:
60+
repository: unicode-org/unicodetools
61+
path: unicodetools/mine/src
62+
- name: Checkout CLDR
63+
uses: actions/checkout@v2
64+
with:
65+
repository: unicode-org/cldr
66+
path: cldr/mine/src
67+
- name: Set up JDK 11
68+
uses: actions/setup-java@v1
69+
with:
70+
java-version: 11
71+
- name: Cache local Maven repository
72+
uses: actions/cache@v2
73+
with:
74+
path: ~/.m2/repository
75+
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
76+
restore-keys: |
77+
${{ runner.os }}-maven-
78+
- name: Set up out-of-source output dir
79+
run: |
80+
mkdir -p unicodetools/mine/Generated/BIN
81+
- name: Initialize Maven settings for all Maven commands
82+
run: |
83+
cd unicodetools/mine/src
84+
mvn -s .github/workflows/mvn-settings.xml -B compile install -DskipTests=true
85+
env:
86+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
87+
- name: Run command - Make Unicode Files
88+
run: |
89+
cd unicodetools/mine/src
90+
mvn -s .github/workflows/mvn-settings.xml exec:java -Dexec.mainClass="org.unicode.text.UCD.Main" -Dexec.args="version 15.0.0 build MakeUnicodeFiles" -pl unicodetools -DCLDR_DIR=$(cd ../../../cldr/mine/src ; pwd) -DUNICODETOOLS_GEN_DIR=$(cd ../Generated ; pwd) -DUNICODETOOLS_REPO_DIR=$(pwd) -DUVERSION=15.0.0
91+
env:
92+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
93+
- name: Run command - Build and Test
94+
run: |
95+
cd unicodetools/mine/src
96+
MAVEN_OPTS="-ea" mvn -s .github/workflows/mvn-settings.xml package -DCLDR_DIR=$(cd ../../../cldr/mine/src ; pwd) -DUNICODETOOLS_GEN_DIR=$(cd ../Generated ; pwd) -DUNICODETOOLS_REPO_DIR=$(pwd) -DUVERSION=15.0.0
97+
env:
98+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
99+

‎docs/build.md‎

Lines changed: 12 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -72,32 +72,19 @@ git clone https://github.com/unicode-org/cldr.git
7272

7373
##### Notes for both out-of-source and in-source build workspaces
7474

75-
Currently, some tests run on the generated output files of a tool (ex: in order to test the validity of the output files). However, after converting these tests into standard JUnit tests, these unit tests are then run in isolation by default. Until we amend the code for such tests, those tests will fail by default. These tests are picked up and run by `mvn test`, and thus, by any other subsequent Maven target in the Maven lifecycle (ex: `package`, `install`).
76-
77-
For people who need `mvn test` or other subsequent Maven targets to succeed, a temporary workaround would be the following (which points the generated files directory to find the test input files from the repository sources):
78-
79-
For out-of-source builds:
80-
```
81-
ln -s ../../src/unicodetools/data/security unicodetools/mine/Generated/security
82-
```
83-
For in-source builds:
84-
```
85-
cd <unicodetools-repo-root>; ln -s unicodetools/data/security output/Generated/security
86-
```
87-
88-
This step to create a symbolic link on the file system is not necessary to run individual tools in Unicode Tools, nor is it intended to last long-term as we refactor code to establish stronger invariants and tests.
75+
Currently, some tests run on the generated output files of a tool (ex: in order to test the validity of the output files). After converting these tests into standard JUnit tests, these unit tests are then run in isolation by default. Our code has been updated to support this behavior because it [now checks](https://github.com/unicode-org/unicodetools/commit/aa6d11c57fe8bbd20484d3a36123c8948e363262) for generated files in the `Generated` directory, and falls back to the repository's checked-in version when a command does not invoke the generation of a new version.
8976

9077
#### Java System properties used in Unicode Tools
9178

9279
(Note: The following example values for Java system properties are paths to local working copies that are organized using the out-of-source build workspace layout, as described above.)
9380

94-
| Property | Example Value |
95-
|-------------------------|------------------------------------------------------|
96-
| CLDR_DIR | /usr/local/google/home/mscherer/cldr/mine/src |
97-
| IMAGES_REPO_DIR | /usr/local/google/home/mscherer/images/mine/src |
98-
| UNICODETOOLS_REPO_DIR | /usr/local/google/home/mscherer/unitools/mine/src |
99-
| UNICODETOOLS_OUTPUT_DIR | /usr/local/google/home/mscherer/unitools/mine |
100-
| UVERSION | 14.0.0 |
81+
| Property | Example Value |
82+
|-------------------------|----------------------------------------------------------------|
83+
| CLDR_DIR | /usr/local/google/home/mscherer/cldr/mine/src |
84+
| IMAGES_REPO_DIR | /usr/local/google/home/mscherer/images/mine/src |
85+
| UNICODETOOLS_REPO_DIR | /usr/local/google/home/mscherer/unitools/mine/src |
86+
| UNICODETOOLS_GEN_DIR | /usr/local/google/home/mscherer/unitools/mine/Generated |
87+
| UVERSION | 14.0.0 |
10188

10289

10390
### Eclipse-specific Additional Setup
@@ -160,12 +147,12 @@ mvn -s .github/workflows/mvn-settings.xml -B compile
160147
Common tasks for Unicode Tools are listed below with example CLI commands with example argument values that they need:
161148

162149
- Make Unicode Files:
163-
* Out-of-source build: `mvn -s .github/workflows/mvn-settings.xml exec:java -Dexec.mainClass="org.unicode.text.UCD.Main" -Dexec.args="version 14.0.0 build MakeUnicodeFiles" -pl unicodetools -DCLDR_DIR=$(cd ../../../cldr/mine/src ; pwd) -DUNICODETOOLS_OUTPUT_DIR=$(cd .. ; pwd) -DUNICODETOOLS_REPO_DIR=$(pwd) -DUVERSION=14.0.0`
164-
* In-source build: `MAVEN_OPTS="-ea" mvn exec:java -Dexec.mainClass="org.unicode.text.UCD.Main" -Dexec.args="version 14.0.0 build MakeUnicodeFiles" -pl unicodetools -DCLDR_DIR=$(cd ../cldr ; pwd) -DUNICODETOOLS_OUTPUT_DIR=$(pwd) -DUNICODETOOLS_REPO_DIR=$(pwd) -DUVERSION=14.0.0`
150+
* Out-of-source build: `mvn -s .github/workflows/mvn-settings.xml exec:java -Dexec.mainClass="org.unicode.text.UCD.Main" -Dexec.args="version 14.0.0 build MakeUnicodeFiles" -pl unicodetools -DCLDR_DIR=$(cd ../../../cldr/mine/src ; pwd) -DUNICODETOOLS_GEN_DIR=$(cd ../Generated ; pwd) -DUNICODETOOLS_REPO_DIR=$(pwd) -DUVERSION=14.0.0`
151+
* In-source build: `MAVEN_OPTS="-ea" mvn exec:java -Dexec.mainClass="org.unicode.text.UCD.Main" -Dexec.args="version 14.0.0 build MakeUnicodeFiles" -pl unicodetools -DCLDR_DIR=$(cd ../cldr ; pwd) -DUNICODETOOLS_GEN_DIR=$(cd output/Generated; pwd) -DUNICODETOOLS_REPO_DIR=$(pwd) -DUVERSION=14.0.0`
165152

166153
- Build and Test:
167-
* Out-of-source build: `MAVEN_OPTS="-ea" mvn package -DCLDR_DIR=$(cd ../../../cldr/mine/src ; pwd) -DUNICODETOOLS_OUTPUT_DIR=$(cd .. ; pwd) -DUNICODETOOLS_REPO_DIR=$(pwd) -DUVERSION=14.0.0`
168-
* In-source build: `MAVEN_OPTS="-ea" mvn package -DCLDR_DIR=$(cd ../cldr ; pwd) -DUNICODETOOLS_OUTPUT_DIR=$(pwd) -DUNICODETOOLS_REPO_DIR=$(pwd) -DUVERSION=14.0.0`
154+
* Out-of-source build: `MAVEN_OPTS="-ea" mvn package -DCLDR_DIR=$(cd ../../../cldr/mine/src ; pwd) -DUNICODETOOLS_GEN_DIR=$(cd ../Generated ; pwd) -DUNICODETOOLS_REPO_DIR=$(pwd) -DUVERSION=14.0.0`
155+
* In-source build: `MAVEN_OPTS="-ea" mvn package -DCLDR_DIR=$(cd ../cldr ; pwd) -DUNICODETOOLS_GEN_DIR=$(cd output/Generated; pwd) -DUNICODETOOLS_REPO_DIR=$(pwd) -DUVERSION=14.0.0`
169156

170157
### Updating CLDR and ICU versions
171158

‎pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,7 @@
102102
<java.awt.headless>true</java.awt.headless>
103103
<UNICODETOOLS_REPO_DIR>${project.basedir}/..</UNICODETOOLS_REPO_DIR>
104104
<!-- sibling of unicodetools-->
105-
<UNICODETOOLS_OUTPUT_DIR>${project.basedir}/../output</UNICODETOOLS_OUTPUT_DIR>
105+
<UNICODETOOLS_GEN_DIR>${project.basedir}/../Generated</UNICODETOOLS_GEN_DIR>
106106
</systemPropertyVariables>
107107
<argLine>-Xmx10g -enableassertions</argLine>
108108
</configuration>

‎unicodetools/src/main/java/org/unicode/draft/CharacterFrequency.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -19,10 +19,10 @@
1919
* For a program that uses this, see ScriptPopulation
2020
*/
2121
public class CharacterFrequency {
22-
public static final String DATA_DIR = Settings.Output.UNICODETOOLS_OUTPUT_DIR +
23-
"DATA/frequency/languages/";
24-
private static final String DATA_DIR_RANK = Settings.Output.UNICODETOOLS_OUTPUT_DIR +
25-
"DATA/frequency/languages-rank/";
22+
public static final String DATA_DIR = Settings.Output.GEN_DIR +
23+
"frequency/languages/";
24+
private static final String DATA_DIR_RANK = Settings.Output.GEN_DIR +
25+
"frequency/languages-rank/";
2626
public static final boolean DEBUG = false;
2727
// static final int MAX_LINE_COUNT = Integer.MAX_VALUE; // 10000;
2828
// static final int MAX_SEQUENCE_CHARS = 15;

‎unicodetools/src/main/java/org/unicode/draft/WebpageCharacterData.java‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@ public static void main(String[] args) throws IOException {
7272

7373
static public void doData() throws IOException {
7474
final BufferedReader in = FileUtilities.openUTF8Reader(
75-
Settings.Output.UNICODETOOLS_OUTPUT_DIR + "DATA/frequency/", SOURCE_DATA);
75+
Settings.Output.GEN_DIR + "frequency/", SOURCE_DATA);
7676
int lineCounter = 0;
7777
final int zeroCountLines = 0;
7878
final HashMap<String, String> langSeen = new HashMap<String,String>();
@@ -112,11 +112,11 @@ static public void doData() throws IOException {
112112
in.close();
113113
System.out.println("Writing data");
114114
//System.out.println("zeroCountLines " + zeroCountLines);
115-
writeData(lang2chars, Settings.Output.UNICODETOOLS_OUTPUT_DIR +
116-
"DATA/frequency/languages");
115+
writeData(lang2chars, Settings.Output.GEN_DIR +
116+
"frequency/languages");
117117
System.out.println("Writing ranked data");
118-
writeData(lang2charsPageRank, Settings.Output.UNICODETOOLS_OUTPUT_DIR +
119-
"DATA/frequency/languages-rank");
118+
writeData(lang2charsPageRank, Settings.Output.GEN_DIR +
119+
"frequency/languages-rank");
120120
}
121121

122122
public static void writeData(Map<String, Counter<Integer>> map, String directory) throws IOException {

‎unicodetools/src/main/java/org/unicode/text/tools/IcannMsr.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@
3636
import com.ibm.icu.util.ULocale;
3737

3838
public class IcannMsr {
39-
private static final String ICANN_DIR = Settings.Output.UNICODETOOLS_OUTPUT_DIR + "DATA/icann/";
39+
private static final String ICANN_DIR = Settings.Output.GEN_DIR + "icann/";
4040

4141
// Change these with new versions
4242

‎unicodetools/src/main/java/org/unicode/text/tools/WordFrequency.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ public class WordFrequency {
2424
static long total;
2525
static {
2626
Splitter tab = Splitter.on("\t").trimResults();
27-
for (String line : FileUtilities.in(Settings.Output.UNICODETOOLS_OUTPUT_DIR + "data/words/", "freq.txt")) {
27+
for (String line : FileUtilities.in(Settings.Output.GEN_DIR + "words/", "freq.txt")) {
2828
List<String> list = tab.splitToList(line);
2929
long count = Long.parseLong(list.get(1));
3030
data.put(list.get(0), count);

‎unicodetools/src/main/java/org/unicode/text/utility/Settings.java‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -258,18 +258,18 @@ public static final class Images {
258258

259259
public static final class Output {
260260
/** The root of where we write output files. Most go into a "Generated" sub-folder. */
261-
public static final String UNICODETOOLS_OUTPUT_DIR =
262-
getRequiredPathAndFix("UNICODETOOLS_OUTPUT_DIR");
263-
public static final String GEN_DIR = UNICODETOOLS_OUTPUT_DIR + "Generated/";
261+
public static final String GEN_DIR =
262+
getRequiredPathAndFix("UNICODETOOLS_GEN_DIR");
264263
public static final String BIN_DIR = GEN_DIR + "BIN/";
265264
public static final String GEN_UCD_DIR = GEN_DIR + "UCD/";
266265
public static final String GEN_UCA_DIR = GEN_DIR + "UCA/";
266+
267267
/**
268268
* Make sure the output dirs exist
269269
*/
270270
public static void ensureOutputDirs() throws FileNotFoundException {
271-
if (!(new File(UNICODETOOLS_OUTPUT_DIR)).isDirectory()) {
272-
throw new FileNotFoundException("Not a directory: UNICODETOOLS_OUTPUT_DIR=" + UNICODETOOLS_OUTPUT_DIR);
271+
if (!(new File(GEN_DIR)).isDirectory()) {
272+
throw new FileNotFoundException("Not a directory: UNICODETOOLS_GEN_DIR=" + GEN_DIR);
273273
}
274274
ensureOutputDir(GEN_DIR);
275275
ensureOutputDir(GEN_UCD_DIR);

‎unicodetools/src/main/java/org/unicode/tools/Ids.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1264,7 +1264,7 @@ private static void load() {
12641264
CharacterIds [] biggest = new CharacterIds[50];
12651265
Counter<Integer> counter = new Counter<>();
12661266

1267-
for (String line : FileUtilities.in(Settings.Output.UNICODETOOLS_OUTPUT_DIR + "DATA/ids/", SOURCE_IDS)) {
1267+
for (String line : FileUtilities.in(Settings.Output.GEN_DIR + "ids/", SOURCE_IDS)) {
12681268
// U+3FCD 㿍 ⿸疒解
12691269
if (line.startsWith("#") || line.startsWith(";")) {
12701270
line = line.substring(1).trim();
@@ -1379,7 +1379,7 @@ private static void load() {
13791379
private static Map<String,String> getMacros() {
13801380
Relation<String,String> temp = Relation.of(new HashMap(), HashSet.class);
13811381
// fetch the data
1382-
for (String line : FileUtilities.in(Settings.Output.UNICODETOOLS_OUTPUT_DIR + "DATA/ids/", SOURCE_IDS)) {
1382+
for (String line : FileUtilities.in(Settings.Output.GEN_DIR + "ids/", SOURCE_IDS)) {
13831383
// CDP-8DA8 &CDP-8DA8; ⿻廿木 ⿱丗木 ⿱𠀍木
13841384
// U+3022 〢 ⿰丨丨
13851385
// CDP-8DBA &CDP-8DBA; &CDP-8DBA;

0 commit comments

Comments
 (0)