Make GenerateEmojiData work as intended on Windows, and run it in CI (#1251)

eggrobin · web-flow · commit 589f08e5879c · 2025-12-03T20:47:13.000+01:00
* Make replaceDifferentOrDelete work on Windows

* Update the documentation

* GenerateEmojiData in CI

* GenerateEmojiData
diff --git a/.github/workflows/cli-build-instructions.yml b/.github/workflows/cli-build-instructions.yml
@@ -165,6 +165,29 @@ jobs:
             exit 1
           }
 
+      - name: Run command - Generate Emoji Data
+        run: |
+          cd unicodetools/mine/src
+          mvn -s .github/workflows/mvn-settings.xml compile exec:java -Dexec.mainClass="org.unicode.tools.emoji.GenerateEmojiData" -am -pl unicodetools  -DCLDR_DIR=$(cd ../../../cldr/mine/src ; pwd)  -DUNICODETOOLS_GEN_DIR=$(cd ../Generated ; pwd)  -DUNICODETOOLS_REPO_DIR=$(pwd)
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      # Some emoji files are UCD files, so we should check that their generator
+      # not only runs, but that it doesn’t change anything.
+      - name: Check that Emoji files are consistent
+        run: |
+          cd unicodetools/mine/src
+          git diff --compact-summary --exit-code || {
+            git diff --compact-summary |
+              awk '{
+                if (previous) {
+                  print "::error file="previous",title=File must be regenerated::Run org.unicode.tools.emoji.GenerateEmojiData."
+                }
+                previous=$1
+              }'
+            exit 1
+          }
+
       # Only test once we know the UCD is internally consistent.
       # MakeUnicodeFiles is much faster than this anyway.
       - name: Run command - Build and Test
diff --git a/docs/pipeline.md b/docs/pipeline.md
@@ -52,8 +52,6 @@ Indic scripts only:
 If the change affects emoji properties, including reserved Extended_Pictographic codepoints that should no longer be 
 reserved:
 - [ ] [GenerateEmojiData](#generateemojidata)
-- [ ] Merge changes from the temporary files that are generated in unicodetools/data/ucd.
-- [ ] Delete the temporary files that aren't needed from unicodetools/data/ucd and unicodetools/data/emoji.
 - [ ] Commit
 
 ---
diff --git a/unicodetools/data/emoji/dev/emoji-sequences.txt b/unicodetools/data/emoji/dev/emoji-sequences.txt
@@ -1,11 +1,11 @@
 # emoji-sequences.txt
-# Date: 2025-07-25, 17:54:32 GMT
+# Date: 2025-12-03, 13:29:04 GMT
 # © 2025 Unicode®, Inc.
 # Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries.
 # For terms of use and license, see https://www.unicode.org/terms_of_use.html
 #
 # Emoji Sequence Data for UTS #51
-# Version: 17.0
+# Version: 18.0
 #
 # For documentation and usage, see https://www.unicode.org/reports/tr51
 #
diff --git a/unicodetools/data/emoji/dev/emoji-test.txt b/unicodetools/data/emoji/dev/emoji-test.txt
@@ -1,11 +1,11 @@
 # emoji-test.txt
-# Date: 2025-08-04, 20:55:31 GMT
+# Date: 2025-12-03, 13:29:04 GMT
 # © 2025 Unicode®, Inc.
 # Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries.
 # For terms of use and license, see https://www.unicode.org/terms_of_use.html
 #
 # Emoji Keyboard/Display Test Data for UTS #51
-# Version: 17.0
+# Version: 18.0
 #
 # For documentation and usage, see https://www.unicode.org/reports/tr51
 #
diff --git a/unicodetools/data/emoji/dev/emoji-zwj-sequences.txt b/unicodetools/data/emoji/dev/emoji-zwj-sequences.txt
@@ -1,11 +1,11 @@
 # emoji-zwj-sequences.txt
-# Date: 2025-01-08, 04:57:12 GMT
+# Date: 2025-12-03, 13:29:04 GMT
 # © 2025 Unicode®, Inc.
 # Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries.
 # For terms of use and license, see https://www.unicode.org/terms_of_use.html
 #
 # Emoji ZWJ Sequences for UTS #51
-# Version: 17.0
+# Version: 18.0
 #
 # For documentation and usage, see https://www.unicode.org/reports/tr51
 #
diff --git a/unicodetools/data/emoji/dev/internal/emoji-internal.txt b/unicodetools/data/emoji/dev/internal/emoji-internal.txt
@@ -1,11 +1,11 @@
 # emoji-internal.txt
-# Date: 2025-01-08, 04:54:45 GMT
+# Date: 2025-12-03, 13:29:03 GMT
 # © 2025 Unicode®, Inc.
 # Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries.
 # For terms of use and license, see https://www.unicode.org/terms_of_use.html
 #
 # Emoji Data Internal for UTS #51
-# Version: 17.0
+# Version: 18.0
 #
 # For documentation and usage, see https://www.unicode.org/reports/tr51
 #
diff --git a/unicodetools/data/emoji/dev/internal/emoji-proposals.txt b/unicodetools/data/emoji/dev/internal/emoji-proposals.txt
diff --git a/unicodetools/data/emoji/dev/internal/emoji-sequences-nonrgi.txt b/unicodetools/data/emoji/dev/internal/emoji-sequences-nonrgi.txt
@@ -1,11 +1,11 @@
 # emoji-sequences-nonrgi.txt
-# Date: 2025-01-08, 04:54:47 GMT
+# Date: 2025-12-03, 13:29:04 GMT
 # © 2025 Unicode®, Inc.
 # Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries.
 # For terms of use and license, see https://www.unicode.org/terms_of_use.html
 #
 # Emoji Sequence Data — Non-RGI for UTS #51
-# Version: 17.0
+# Version: 18.0
 #
 # For documentation and usage, see https://www.unicode.org/reports/tr51
 #
diff --git a/unicodetools/data/emoji/dev/internal/emoji-zwj-sequences-nonrgi.txt b/unicodetools/data/emoji/dev/internal/emoji-zwj-sequences-nonrgi.txt
@@ -1,11 +1,11 @@
 # emoji-zwj-sequences-nonrgi.txt
-# Date: 2025-01-08, 04:54:47 GMT
+# Date: 2025-12-03, 13:29:04 GMT
 # © 2025 Unicode®, Inc.
 # Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries.
 # For terms of use and license, see https://www.unicode.org/terms_of_use.html
 #
 # Emoji ZWJ Sequences — Non-RGI for UTS #51
-# Version: 17.0
+# Version: 18.0
 #
 # For documentation and usage, see https://www.unicode.org/reports/tr51
 #
diff --git a/unicodetools/data/ucd/dev/emoji/emoji-variation-sequences.txt b/unicodetools/data/ucd/dev/emoji/emoji-variation-sequences.txt
@@ -1,11 +1,11 @@
 # emoji-variation-sequences.txt
-# Date: 2025-01-30, 21:48:29 GMT
+# Date: 2025-12-03, 13:29:04 GMT
 # © 2025 Unicode®, Inc.
 # Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries.
 # For terms of use and license, see https://www.unicode.org/terms_of_use.html
 #
 # Emoji Variation Sequences for UTS #51
-# Version: 17.0
+# Version: 18.0.0
 #
 # For documentation and usage, see https://www.unicode.org/reports/tr51
 #
diff --git a/unicodetools/src/main/java/org/unicode/text/utility/Utility.java b/unicodetools/src/main/java/org/unicode/text/utility/Utility.java
@@ -31,6 +31,7 @@
 import java.io.UnsupportedEncodingException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -1290,7 +1291,7 @@ public static boolean replaceDifferentOrDelete(
                             + lines[1].substring(diff)
                             + "'");
         }
-        new File(newFile).renameTo(oldFile2);
+        Files.move(Path.of(newFile), Path.of(oldFile), StandardCopyOption.REPLACE_EXISTING);
         return true;
     }
 

Original file line number	Diff line number	Diff line change
`@@ -1,11 +1,11 @@`
`1`	`1`	`# emoji-sequences.txt`
`2`		`-# Date: 2025-07-25, 17:54:32 GMT`
	`2`	`+# Date: 2025-12-03, 13:29:04 GMT`
`3`	`3`	`# © 2025 Unicode®, Inc.`
`4`	`4`	`# Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries.`
`5`	`5`	`# For terms of use and license, see https://www.unicode.org/terms_of_use.html`
`6`	`6`	`#`
`7`	`7`	`# Emoji Sequence Data for UTS #51`
`8`		`-# Version: 17.0`
	`8`	`+# Version: 18.0`
`9`	`9`	`#`
`10`	`10`	`# For documentation and usage, see https://www.unicode.org/reports/tr51`
`11`	`11`	`#`
Original file line number	Diff line number	Diff line change
`@@ -1,11 +1,11 @@`
`1`	`1`	`# emoji-test.txt`
`2`		`-# Date: 2025-08-04, 20:55:31 GMT`
	`2`	`+# Date: 2025-12-03, 13:29:04 GMT`
`3`	`3`	`# © 2025 Unicode®, Inc.`
`4`	`4`	`# Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries.`
`5`	`5`	`# For terms of use and license, see https://www.unicode.org/terms_of_use.html`
`6`	`6`	`#`
`7`	`7`	`# Emoji Keyboard/Display Test Data for UTS #51`
`8`		`-# Version: 17.0`
	`8`	`+# Version: 18.0`
`9`	`9`	`#`
`10`	`10`	`# For documentation and usage, see https://www.unicode.org/reports/tr51`
`11`	`11`	`#`
Original file line number	Diff line number	Diff line change
`@@ -1,11 +1,11 @@`
`1`	`1`	`# emoji-zwj-sequences.txt`
`2`		`-# Date: 2025-01-08, 04:57:12 GMT`
	`2`	`+# Date: 2025-12-03, 13:29:04 GMT`
`3`	`3`	`# © 2025 Unicode®, Inc.`
`4`	`4`	`# Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries.`
`5`	`5`	`# For terms of use and license, see https://www.unicode.org/terms_of_use.html`
`6`	`6`	`#`
`7`	`7`	`# Emoji ZWJ Sequences for UTS #51`
`8`		`-# Version: 17.0`
	`8`	`+# Version: 18.0`
`9`	`9`	`#`
`10`	`10`	`# For documentation and usage, see https://www.unicode.org/reports/tr51`
`11`	`11`	`#`
Original file line number	Diff line number	Diff line change
`@@ -1,11 +1,11 @@`
`1`	`1`	`# emoji-internal.txt`
`2`		`-# Date: 2025-01-08, 04:54:45 GMT`
	`2`	`+# Date: 2025-12-03, 13:29:03 GMT`
`3`	`3`	`# © 2025 Unicode®, Inc.`
`4`	`4`	`# Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries.`
`5`	`5`	`# For terms of use and license, see https://www.unicode.org/terms_of_use.html`
`6`	`6`	`#`
`7`	`7`	`# Emoji Data Internal for UTS #51`
`8`		`-# Version: 17.0`
	`8`	`+# Version: 18.0`
`9`	`9`	`#`
`10`	`10`	`# For documentation and usage, see https://www.unicode.org/reports/tr51`
`11`	`11`	`#`
Original file line number	Diff line number	Diff line change
`@@ -1,11 +1,11 @@`
`1`	`1`	`# emoji-sequences-nonrgi.txt`
`2`		`-# Date: 2025-01-08, 04:54:47 GMT`
	`2`	`+# Date: 2025-12-03, 13:29:04 GMT`
`3`	`3`	`# © 2025 Unicode®, Inc.`
`4`	`4`	`# Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries.`
`5`	`5`	`# For terms of use and license, see https://www.unicode.org/terms_of_use.html`
`6`	`6`	`#`
`7`	`7`	`# Emoji Sequence Data — Non-RGI for UTS #51`
`8`		`-# Version: 17.0`
	`8`	`+# Version: 18.0`
`9`	`9`	`#`
`10`	`10`	`# For documentation and usage, see https://www.unicode.org/reports/tr51`
`11`	`11`	`#`
Original file line number	Diff line number	Diff line change
`@@ -1,11 +1,11 @@`
`1`	`1`	`# emoji-zwj-sequences-nonrgi.txt`
`2`		`-# Date: 2025-01-08, 04:54:47 GMT`
	`2`	`+# Date: 2025-12-03, 13:29:04 GMT`
`3`	`3`	`# © 2025 Unicode®, Inc.`
`4`	`4`	`# Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries.`
`5`	`5`	`# For terms of use and license, see https://www.unicode.org/terms_of_use.html`
`6`	`6`	`#`
`7`	`7`	`# Emoji ZWJ Sequences — Non-RGI for UTS #51`
`8`		`-# Version: 17.0`
	`8`	`+# Version: 18.0`
`9`	`9`	`#`
`10`	`10`	`# For documentation and usage, see https://www.unicode.org/reports/tr51`
`11`	`11`	`#`
Original file line number	Diff line number	Diff line change
`@@ -1,11 +1,11 @@`
`1`	`1`	`# emoji-variation-sequences.txt`
`2`		`-# Date: 2025-01-30, 21:48:29 GMT`
	`2`	`+# Date: 2025-12-03, 13:29:04 GMT`
`3`	`3`	`# © 2025 Unicode®, Inc.`
`4`	`4`	`# Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries.`
`5`	`5`	`# For terms of use and license, see https://www.unicode.org/terms_of_use.html`
`6`	`6`	`#`
`7`	`7`	`# Emoji Variation Sequences for UTS #51`
`8`		`-# Version: 17.0`
	`8`	`+# Version: 18.0.0`
`9`	`9`	`#`
`10`	`10`	`# For documentation and usage, see https://www.unicode.org/reports/tr51`
`11`	`11`	`#`