Fix charts & data for emoji 13.1 (#25)

* aug 27 charts

* test

* first cut at fixing emoji

* 2nd pass at emoji fixes

* Fixes to get the counts right

* Drop the 2 provisional characters replaced by sequences; add the handshakes with tones. Note special code to hack the names.

* Updated emoji charts (small changes for proposal, note for handshake). Unrelated utility

* small updates from review; and make sure to use latest images

* Minor comment

* Small change to restore kiss/couple (old code was filtering them out); also two unrelated changes to remove errors in UnicodeJsps.

* drop old provisional mending heart

* Cleanup proposal data, fix Emoji VERSION14

Author: macchiati

UnicodeJsps/build/.gitignore

@@ -0,0 +1 @@
+/classes/

UnicodeJspsTest/src/org/unicode/jsptest/TestUnicodeSet.java

@@ -136,7 +136,7 @@ public void TestPretty() {
     public void TestU60 () {
         logln("ICU Version: " + VersionInfo.ICU_VERSION.toString());
         logln("Unicode Data Version:   " + UCharacter.getUnicodeVersion().toString());
-        logln("Java Version:   " + VersionInfo.javaVersion().toString());
+//        logln("Java Version:   " + VersionInfo.javaVersion().toString());
         logln("CLDR Data Version:      " + LocaleData.getCLDRVersion().toString());
         logln("Time Zone Data Version: " + TimeZone.getTZDataVersion());
 

unicodetools/data/emoji/13.1/emoji-sequences.txt

@@ -1388,6 +1388,15 @@
 1F9DD 1F3FE   ; RGI_Emoji_Modifier_Sequence  ; elf: medium-dark skin tone                                     # E5.0   [1] (🧝🏾)
 1F9DD 1F3FF   ; RGI_Emoji_Modifier_Sequence  ; elf: dark skin tone                                            # E5.0   [1] (🧝🏿)
 
-# Total elements: 580
+1F491 1F3FB    ; RGI_Emoji_Modifier_Sequence
+1F491 1F3FC    ; RGI_Emoji_Modifier_Sequence
+1F491 1F3FD    ; RGI_Emoji_Modifier_Sequence
+1F491 1F3FE    ; RGI_Emoji_Modifier_Sequence
+1F491 1F3FF    ; RGI_Emoji_Modifier_Sequence
+1F48F 1F3FB    ; RGI_Emoji_Modifier_Sequence
+1F48F 1F3FC    ; RGI_Emoji_Modifier_Sequence
+1F48F 1F3FD    ; RGI_Emoji_Modifier_Sequence
+1F48F 1F3FE    ; RGI_Emoji_Modifier_Sequence
+1F48F 1F3FF    ; RGI_Emoji_Modifier_Sequence
 
 #EOF

unicodetools/data/emoji/13.1/emoji-zwj-sequences.txt

@@ -46,7 +46,7 @@ public void TestAEncodings() {
     public void TestU60 () {
         logln("ICU Version: " + VersionInfo.ICU_VERSION.toString());
         logln("Unicode Data Version:   " + UCharacter.getUnicodeVersion().toString());
-        logln("Java Version:   " + VersionInfo.javaVersion().toString());
+//        logln("Java Version:   " + VersionInfo.javaVersion().toString());
         logln("CLDR Data Version:      " + LocaleData.getCLDRVersion().toString());
         logln("Time Zone Data Version: " + TimeZone.getTZDataVersion());
 

unicodetools/org/unicode/jsptest/TestUnicodeSet.java

@@ -0,0 +1,14 @@
+package org.unicode.propstest;
+
+import com.ibm.icu.text.UnicodeSet;
+
+public class PrintUnicodeSet {
+    public static void main(String[] args) {
+	System.out.println(new UnicodeSet(
+		"[‾‽‸⁂↚↛↮↙↜↝↞↟↠↡↢↣↤↥↦↧↨↫↬↭↯↰↱↲↳↴↵↶↷↸↹↺↻↼↽↾↿⇀⇁⇂⇃⇄⇇⇈⇉⇊⇋⇌⇐⇍"
+			+ "⇑⇒⇏⇓⇔⇎⇖⇗⇘⇙⇚⇛⇜⇝⇞⇟⇠⇡⇢⇣⇤⇥⇦⇧⇨⇩⇪⇵∀∂∃∅∉∋∎∏∑≮≯∓∕⁄∗∘∙∝∟∠∣∥∧∫∬∮∴∵∶∷∼∽∾"
+			+ "≃≅≌≒≖≣≦≧≪≫≬≳≺≻⊁⊃⊆⊇⊕⊖⊗⊘⊙⊚⊛⊞⊟⊥⊮⊰⊱⋭⊶⊹⊿⋁⋂⋃⋅⋆⋈⋒⋘⋙⋮⋯⋰⋱■□▢▣▤▥▦▧▨▩▬▭▮▰△▴▵▷▸▹►▻▽▾"
+			+ "▿◁◂◃◄◅◆◇◈◉◌◍◎◐◑◒◓◔◕◖◗◘◙◜◝◞◟◠◡◢◣◤◥◦◳◷◻◽◿⨧⨯⨼⩣⩽⪍⪚⪺₢₣₤₰₳₶₷₨﷼]"
+		).toPattern(false));
+    }
+}

unicodetools/org/unicode/propstest/PrintUnicodeSet.java

@@ -29,7 +29,7 @@ public class Settings {
     public static final String UNICODEJSPS_DIRECTORY = SVN_WORKSPACE_DIRECTORY + "UnicodeJsps/";
     public static final String UNICODE_DRAFT_DIRECTORY =
             CldrUtility.getProperty("UNICODE_DRAFT_DIR",
-                    SVN_WORKSPACE_DIRECTORY + "unicode-draft") + '/';
+                    SVN_WORKSPACE_DIRECTORY + "emoji/docs") + '/';
     public static final String UNICODE_DRAFT_PUBLIC = UNICODE_DRAFT_DIRECTORY + "Public/";
 
     public static final String DATA_DIR = Utility.fixFileName(CldrUtility.getProperty("UCD_DIR", UNICODETOOLS_DIRECTORY + "data/")) + "/";

unicodetools/org/unicode/text/utility/Settings.java

@@ -0,0 +1,17 @@
+package org.unicode.tools;
+
+import java.util.Objects;
+
+import com.ibm.icu.text.UnicodeSet;
+
+public class CompareUnicodeSets {
+public static void main(String[] args) {
+    UnicodeSet a = new UnicodeSet("[ا أ آ ب پ ت ٹ ث ج چ ح خ د ڈ ذ ر ڑ ز ژ س ش ص ض ط ظ ع غ ف ق ک گ ل م ن ں و ؤ ہ ۂ ھ ء ی ئ ے ة ه]");
+    UnicodeSet b = new UnicodeSet("[ا ب پ ت ٹ ث ج چ ح خ د ڈ ذ ر ڑ ز ژ س ش ص ض ط ظ ع غ ف ق ک گ ل م ن و ہ ھ ء ی ے]");
+    System.out.println("old:\t" + a.toPattern(false));
+    System.out.println("new:\t" + b.toPattern(false));
+    System.out.println("old-only:\t" + new UnicodeSet(a).removeAll(b).toPattern(false));
+    System.out.println("new-only:\t" + new UnicodeSet(b).removeAll(a).toPattern(false));
+    Objects.equals(3, 3);
+}
+}

unicodetools/org/unicode/tools/CompareUnicodeSets.java

@@ -53,6 +53,11 @@
 import com.ibm.icu.util.ULocale;
 import com.ibm.icu.util.VersionInfo;
 
+/**
+ * Provides data for candidates, reading the file candidateData.txt.
+ * Note: At the end of a release, before the Draft Candidates are retired, run CandidateData.java to get the proposals for those
+ * candidates, and add to the end of proposalData.txt
+ */
 public class CandidateData implements Transform<String, String>, EmojiDataSource {
     private static final String TEST_STRING = "👩‍🤝‍👩";
     private static final boolean SHOW_COMBOS = false;
@@ -128,6 +133,7 @@ public Set<Comparable> make() {
     private final UnicodeMap<Status> statuses = new UnicodeMap<>();
     private final UnicodeSet singleCharacters = new UnicodeSet();
     private final UnicodeSet allCharacters = new UnicodeSet();
+    private final UnicodeSet fullDraftForProposals;
     private final UnicodeSet allNonProvisional = new UnicodeSet();
     private final UnicodeSet textPresentation = new UnicodeSet();
     private UnicodeSet provisional = new UnicodeSet();
@@ -305,6 +311,7 @@ private CandidateData(String sourceFile) {
 	//	allCharacters.removeAll(singleCharacters);
 
 	allCharacters.removeAll(duplicates).freeze();
+	fullDraftForProposals = new UnicodeSet(statuses.getSet(Status.Draft_Candidate)).freeze();
 	statuses.removeAll(duplicates).freeze();
 	comments.freeze();
 	categories.freeze();
@@ -424,7 +431,9 @@ private void addHackName(String name, String singleton, String prefix, String su
 		    compoundName += ", " + jName;
 		}
 		names.put(sequence, compoundName);
-		System.out.println(Utility.hex(sequence) + " => " + compoundName);
+		if (DEBUG) {
+		    System.out.println(Utility.hex(sequence) + " => " + compoundName);
+		}
 	    }
 	}
     }
@@ -797,7 +806,20 @@ public String getCategory(int source) {
     }
     public String getCategory(String source) {
 	String result = EmojiOrder.STD_ORDER.charactersToOrdering.get(source);
-	return result != null ? result : categories.get(source);
+	if (result != null) {
+	    return result;
+	}
+//	final String stripped = EmojiData.removeEmojiVariants(EmojiData.MODIFIERS.stripFrom(source, true));
+//	result = EmojiOrder.STD_ORDER.charactersToOrdering.get(stripped);
+//	if (result != null) {
+//	    return result;
+//	}
+	result = categories.get(source);
+	if (result != null) {
+	    return result;
+	}
+	System.out.println(Utility.hex(source) + "; " + source);
+	return null;
     }
 
     public List<Integer> getOrder() {
@@ -825,7 +847,7 @@ public static void main(String[] args) {
 	DEBUG = true;
 	CandidateData candidateData = CandidateData.getInstance();
 	if (args.length == 0) {
-	    throw new IllegalArgumentException();
+	    args = new String[] {"proposals"};
 	}
 	int count = 0;
 	for (String arg : args) {
@@ -896,10 +918,10 @@ private static void generateProposalData(CandidateData instance) {
 	//1F931;  L2/16-280,L2/16-282r;   BREAST-FEEDING   
 	Set<String> done = new HashSet<>();
 	UnicodeSet missing = new UnicodeSet();
-	for (String item : instance.allCharacters) {
+	for (String item : instance.fullDraftForProposals) {
 	    if (instance.statuses.get(item) == Status.Provisional_Candidate
-		    || EmojiData.MODIFIERS.containsSome(item)
-		    || Emoji.GENDER_MARKERS.containsSome(item)
+//		    || EmojiData.MODIFIERS.containsSome(item)
+//		    || Emoji.GENDER_MARKERS.containsSome(item)
 		    ) {
 		continue;
 	    }
@@ -909,9 +931,9 @@ private static void generateProposalData(CandidateData instance) {
 	    }
 	    done.add(skeleton);
 	    Set<String> proposals = instance.getProposal(item);
-	    if (proposals == null) {
-		missing.add(item);
-	    }
+//	    if (proposals == null) {
+//		missing.add(item);
+//	    }
 	    System.out.println(Utility.hex(skeleton)
 		    + "; " + CollectionUtilities.join(proposals, ", ")
 		    + "; " + instance.getName(item));

unicodetools/org/unicode/tools/emoji/CandidateData.java

@@ -62,6 +62,7 @@ public class Emoji {
     /**
      * Constants for versions
      */
+    public static final VersionInfo VERSION14 = VersionInfo.getInstance(14,0);
     public static final VersionInfo VERSION13_1 = VersionInfo.getInstance(13,1);
     public static final VersionInfo VERSION13 = VersionInfo.getInstance(13);
     public static final VersionInfo VERSION12_1 = VersionInfo.getInstance(12,1);
@@ -91,9 +92,9 @@ public class Emoji {
      * Change each following once we release. That is, VERSION_LAST_RELEASED* becomes VERSION_BETA*, and both the latter increment.
      * Also add to EMOJI_TO_UNICODE_VERSION
      */
-    public static final VersionInfo VERSION_LAST_RELEASED2 = VERSION12_1;
-    public static final VersionInfo VERSION_LAST_RELEASED = VERSION13;
-    public static final VersionInfo VERSION_BETA = VERSION13_1;
+    public static final VersionInfo VERSION_LAST_RELEASED2 = VERSION13;
+    public static final VersionInfo VERSION_LAST_RELEASED = VERSION13_1;
+    public static final VersionInfo VERSION_BETA = VERSION14;
 
     public static final VersionInfo VERSION_TO_TEST = VERSION_BETA;
     public static final VersionInfo VERSION_TO_TEST_PREVIOUS = VERSION_LAST_RELEASED;
@@ -171,7 +172,7 @@ public class Emoji {
 
     public static final VersionInfo VERSION_TO_GENERATE_UNICODE = IS_BETA ? VERSION_BETA_UNICODE : VERSION_LAST_RELEASED_UNICODE;
 
-    public static final String TR51_SVN_DIR = Settings.UNICODE_DRAFT_DIRECTORY + "reports/tr51/";
+    //public static final String TR51_SVN_DIR = Settings.UNICODE_DRAFT_DIRECTORY + "reports/tr51/";
     //public static final String TR51_PREFIX = IS_BETA ? "internal-beta/" : "internal/";
 
     public static final String EMOJI_DIR = Settings.UNICODE_DRAFT_DIRECTORY + "emoji/" + (Emoji.ABBR ? "🏴" : "");
@@ -185,7 +186,7 @@ public class Emoji {
     public static final String DATA_DIR_PRODUCTION = DATA_DIR_PRODUCTION_BASE + VERSION_STRING + "/";
 
     public static final String IMAGES_SOURCE_DIR_SVG = Settings.UNICODETOOLS_DIRECTORY + "data/images/";
-    static final String IMAGES_OUTPUT_DIR = TR51_SVN_DIR + "images/";
+    public static final String IMAGES_OUTPUT_DIR = Settings.UNICODETOOLS_DIRECTORY + "../../images/emoji/";
 
     public enum ModifierStatus {
 	none, modifier, modifier_base;