Added support for kRSUnicode/Unicode_Radical_Stroke.

GoldenCrystal · GoldenCrystal · commit 475e9c5e44b7 · 2014-11-24T02:36:09.000+01:00
diff --git a/README.md b/README.md
@@ -123,6 +123,7 @@ This library includes a subset of the official [Unicode Character Database](http
 * ID_Continue
 * XID_Start
 * XID_Continue
+* Unicode_Radical_Stroke (This is actually kRSUnicode from the Unihan database)
 * Code point cross references extracted from NamesList.txt
 
 NB: The UCD property ISO_Comment will never be included since this one is empty in all new Unicode versions.
@@ -131,6 +132,7 @@ NB: The UCD property ISO_Comment will never be included since this one is empty
 * kAccountingNumeric
 * kOtherNumeric
 * kPrimaryNumeric
+* kRSUnicode
 * kDefinition
 * kMandarin
 * kCantonese
diff --git a/UnicodeInformation.Builder/Program.cs b/UnicodeInformation.Builder/Program.cs
@@ -32,6 +32,7 @@ internal class Program
 			"Unihan_NumericValues.txt",
 			"Unihan_Readings.txt",
 			"Unihan_Variants.txt",
+			"Unihan_IRGSources.txt",
 		};
 
 		private static HttpMessageHandler httpMessageHandler;
diff --git a/UnicodeInformation.Builder/UnicodeDataProcessor.cs b/UnicodeInformation.Builder/UnicodeDataProcessor.cs
@@ -19,6 +19,7 @@ internal class UnicodeDataProcessor
 		public const string UnihanReadingsFileName = "Unihan_Readings.txt";
 		public const string UnihanVariantsFileName = "Unihan_Variants.txt";
 		public const string UnihanNumericValuesFileName = "Unihan_NumericValues.txt";
+		public const string UnihanIrgSourcesFileName = "Unihan_IRGSources.txt";
 
 		private static string ParseSimpleCaseMapping(string mapping)
 		{
@@ -45,6 +46,7 @@ public static async Task<UnicodeInfoBuilder> BuildDataAsync(IDataSource ucdSourc
 			await ProcessUnihanReadings(unihanSource, builder).ConfigureAwait(false);
 			await ProcessUnihanVariants(unihanSource, builder).ConfigureAwait(false);
 			await ProcessUnihanNumericValues(unihanSource, builder).ConfigureAwait(false);
+			await ProcessUnihanIrgSources(unihanSource, builder).ConfigureAwait(false);
 
 			return builder;
 		}
@@ -438,5 +440,48 @@ private static async Task ProcessUnihanNumericValues(IDataSource unihanDataSourc
 				}
 			}
 		}
+
+		private static async Task ProcessUnihanIrgSources(IDataSource unihanDataSource, UnicodeInfoBuilder builder)
+		{
+			using (var reader = new UnihanDataFileReader(await unihanDataSource.OpenDataFileAsync(UnihanIrgSourcesFileName).ConfigureAwait(false)))
+			{
+				while (reader.Read())
+				{
+					switch (reader.PropertyName)
+					{
+						case UnihanProperty.kRSUnicode:
+							var entry = builder.GetUnihan(reader.CodePoint);
+							var values = reader.PropertyValue.Split(' ');
+
+                            foreach (var value in values)
+							{
+								bool isSimplified = false;
+								int index;
+
+								for (int i = 0; i < value.Length; ++i)
+								{
+									switch (value[i])
+									{
+										case '\'':
+											isSimplified = true;
+											goto case '.';
+										case '.':
+											index = i;
+											goto SeparatorFound;
+									}
+								}
+								throw new InvalidDataException("Failed to decode value for kRSUnicode / Unicode_Radical_Stroke.");
+
+							SeparatorFound: ;
+								entry.UnicodeRadicalStrokeCounts.Add(new UnicodeRadicalStrokeCount(byte.Parse(value.Substring(0, index), NumberStyles.None), byte.Parse(value.Substring(index + (isSimplified ? 2 : 1)), NumberStyles.None), isSimplified));
+							}
+                            break;
+						default:
+							// Ignore unhandled properties for now.
+							break;
+					}
+				}
+			}
+		}
 	}
 }
diff --git a/UnicodeInformation.Builder/UnihanCharacterDataBuilder.cs b/UnicodeInformation.Builder/UnihanCharacterDataBuilder.cs
@@ -23,6 +23,8 @@ internal sealed class UnihanCharacterDataBuilder
 		private string simplifiedVariant;
 		private string traditionalVariant;
 
+		private readonly List<UnicodeRadicalStrokeCount> unicodeRadicalStrokeCounts = new List<UnicodeRadicalStrokeCount>();
+
 		public int CodePoint { get { return codePoint; } }
 		public UnihanNumericType NumericType { get { return numericType; } set { numericType = value; } }
 		public long NumericValue { get { return numericValue; } set { numericValue = value; } }
@@ -36,6 +38,7 @@ internal sealed class UnihanCharacterDataBuilder
 		public string VietnameseReading { get { return vietnameseReading; } set { vietnameseReading = value; } }
 		public string SimplifiedVariant { get { return simplifiedVariant; } set { simplifiedVariant = value; } }
 		public string TraditionalVariant { get { return traditionalVariant; } set { traditionalVariant = value; } }
+		public ICollection<UnicodeRadicalStrokeCount> UnicodeRadicalStrokeCounts { get { return unicodeRadicalStrokeCounts; } }
 
 		internal UnihanCharacterDataBuilder(int codePoint)
 		{
@@ -49,6 +52,7 @@ internal UnihanCharacterData ToCharacterData()
 				codePoint,
 				numericType,
 				numericValue,
+				unicodeRadicalStrokeCounts.ToArray(),
 				definition,
 				mandarinReading,
 				cantoneseReading,
@@ -67,6 +71,14 @@ internal void WriteToFile(BinaryWriter writer)
 			UnihanFields fields = default(UnihanFields);
 
 			fields |= (UnihanFields)NumericType;
+			// For now, we have enough bits to encode the length of the array in the field specifier, so we'll do that.
+			// (NB: A quick analysis of the files revealed thare there are almost always exactly one Radical/Stroke count, and occasionally two, yet never more.)
+			if (unicodeRadicalStrokeCounts.Count > 0)
+			{
+				if (unicodeRadicalStrokeCounts.Count == 1) fields |= UnihanFields.UnicodeRadicalStrokeCount;
+				else if (unicodeRadicalStrokeCounts.Count == 2) fields |= UnihanFields.UnicodeRadicalStrokeCountTwice;
+				else fields |= UnihanFields.UnicodeRadicalStrokeCount | UnihanFields.UnicodeRadicalStrokeCountTwice;
+			}
 			if (Definition != null) fields |= UnihanFields.Definition;
 			if (MandarinReading != null) fields |= UnihanFields.MandarinReading;
 			if (CantoneseReading != null) fields |= UnihanFields.CantoneseReading;
@@ -81,6 +93,17 @@ internal void WriteToFile(BinaryWriter writer)
 			writer.Write((ushort)fields);
 
 			writer.WriteCodePoint(UnihanCharacterData.PackCodePoint(codePoint));
+			if ((fields & UnihanFields.UnicodeRadicalStrokeCountMore) != 0)
+			{
+				if ((fields & (UnihanFields.UnicodeRadicalStrokeCountMore)) == UnihanFields.UnicodeRadicalStrokeCountMore)
+					writer.Write(checked((byte)(unicodeRadicalStrokeCounts.Count - 3)));
+
+                foreach (var radicalStrokeCount in unicodeRadicalStrokeCounts)
+				{
+					writer.Write(radicalStrokeCount.Radical);
+					writer.Write((byte)(radicalStrokeCount.StrokeCount | (radicalStrokeCount.IsSimplified ? 0x80 : 0)));
+                }
+			}
 			if ((fields & UnihanFields.OtherNumeric) != 0) writer.Write(numericValue);
 			if ((fields & UnihanFields.Definition) != 0) writer.Write(Definition);
 			if ((fields & UnihanFields.MandarinReading) != 0) writer.Write(MandarinReading);
diff --git a/UnicodeInformation/UnicodeCharInfo.cs b/UnicodeInformation/UnicodeCharInfo.cs
@@ -52,6 +52,8 @@ public struct UnicodeCharInfo
 		public ContributoryProperties ContributoryProperties { get { return unicodeCharacterData?.ContributoryProperties ?? 0; } }
 		public CoreProperties CoreProperties { get { return unicodeCharacterData?.CoreProperties ?? 0; } }
 		public UnicodeCrossReferenceCollection CrossRerefences { get { return new UnicodeCrossReferenceCollection(unicodeCharacterData?.CrossRerefences); } }
+		[ValueName("kRSUnicode"), ValueName("cjkRSUnicode"), ValueName("Unicode_Radical_Stroke"), ValueName("URS")]
+		public UnicodeRadicalStrokeCountCollection UnicodeRadicalStrokeCounts { get { return new UnicodeRadicalStrokeCountCollection(unihanCharacterData?.UnicodeRadicalStrokeCounts); } }
 
 		[ValueName("kDefinition")]
 		public string Definition { get { return unihanCharacterData?.Definition; } }
diff --git a/UnicodeInformation/UnicodeInfo.cs b/UnicodeInformation/UnicodeInfo.cs
@@ -185,6 +185,18 @@ private static UnihanCharacterData ReadUnihanCharacterDataEntry(BinaryReader rea
 				reader.ReadInt64() :
 				0;
 
+			UnicodeRadicalStrokeCount[] unicodeRadicalStrokeCounts = (fields & UnihanFields.UnicodeRadicalStrokeCountMore) != 0 ?
+				new UnicodeRadicalStrokeCount
+				[
+					(fields & UnihanFields.UnicodeRadicalStrokeCountMore) == UnihanFields.UnicodeRadicalStrokeCountMore ?
+						reader.ReadByte() + 3 :
+						((byte)(fields & UnihanFields.UnicodeRadicalStrokeCountMore) >> 2)
+				] :
+				UnicodeRadicalStrokeCount.EmptyArray;
+
+			for (int i = 0; i < unicodeRadicalStrokeCounts.Length; ++i)
+				unicodeRadicalStrokeCounts[i] = new UnicodeRadicalStrokeCount(reader.ReadByte(), reader.ReadByte());
+
 			string definition = (fields & UnihanFields.Definition) != 0 ? reader.ReadString() : null;
 			string mandarinReading = (fields & UnihanFields.MandarinReading) != 0 ? reader.ReadString() : null;
 			string cantoneseReading = (fields & UnihanFields.CantoneseReading) != 0 ? reader.ReadString() : null;
@@ -201,6 +213,7 @@ private static UnihanCharacterData ReadUnihanCharacterDataEntry(BinaryReader rea
 				codePoint,
 				numericType,
 				numericValue,
+				unicodeRadicalStrokeCounts,
 				definition,
 				mandarinReading,
 				cantoneseReading,
diff --git a/UnicodeInformation/UnicodeInformation.csproj b/UnicodeInformation/UnicodeInformation.csproj
@@ -67,6 +67,7 @@
     <Compile Include="CoreProperties.cs" />
     <Compile Include="HangulInfo.cs" />
     <Compile Include="UnicodeCrossReferenceCollection.cs" />
+    <Compile Include="UnicodeRadicalStrokeCountCollection.cs" />
     <Compile Include="UnicodeNameAliasCollection.cs" />
     <Compile Include="UcdFields.cs" />
     <Compile Include="EnumHelper.cs" />
@@ -81,6 +82,7 @@
     <Compile Include="UnicodeInfo.cs" />
     <Compile Include="UnicodeNameAlias.cs" />
     <Compile Include="UnicodeNameAliasKind.cs" />
+    <Compile Include="UnicodeRadicalStrokeCount.cs" />
     <Compile Include="UnihanNumericType.cs" />
     <Compile Include="UnicodeNumericType.cs" />
     <Compile Include="UnihanCharacterData.cs" />
diff --git a/UnicodeInformation/UnicodeRadicalStrokeCount.cs b/UnicodeInformation/UnicodeRadicalStrokeCount.cs
@@ -0,0 +1,34 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace System.Unicode
+{
+	public struct UnicodeRadicalStrokeCount
+	{
+		internal static readonly UnicodeRadicalStrokeCount[] EmptyArray = new UnicodeRadicalStrokeCount[0];
+
+		private readonly byte radical;
+		private readonly byte strokeCount;
+
+		internal UnicodeRadicalStrokeCount(byte rawRadical, byte rawStrokeCount)
+		{
+			radical = rawRadical;
+			strokeCount = rawStrokeCount;
+		}
+
+		internal UnicodeRadicalStrokeCount(byte radical, byte strokeCount, bool isSimplified)
+		{
+			this.radical = radical;
+			this.strokeCount = strokeCount;
+
+			if (isSimplified) this.strokeCount |= 0x80;
+		}
+
+		public byte Radical { get { return radical; } }
+		public byte StrokeCount { get { return strokeCount; } }
+		public bool IsSimplified { get { return (strokeCount & 0x80) != 0; } }
+    }
+}
diff --git a/UnicodeInformation/UnicodeRadicalStrokeCountCollection.cs b/UnicodeInformation/UnicodeRadicalStrokeCountCollection.cs
@@ -0,0 +1,68 @@
+﻿using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace System.Unicode
+{
+	public struct UnicodeRadicalStrokeCountCollection : IList<UnicodeRadicalStrokeCount>
+	{
+		public struct Enumerator : IEnumerator<UnicodeRadicalStrokeCount>
+		{
+			private readonly UnicodeRadicalStrokeCount[] items;
+			private int index;
+
+			internal Enumerator(UnicodeRadicalStrokeCount[] items)
+			{
+				this.items = items;
+				this.index = -1;
+			}
+
+			public void Dispose() { }
+
+			public UnicodeRadicalStrokeCount Current { get { return items[index]; } }
+			object IEnumerator.Current { get { return Current; } }
+
+			public bool MoveNext() { return index < items.Length && ++index < items.Length; }
+
+			void IEnumerator.Reset() { this.index = -1; }
+		}
+
+		private readonly UnicodeRadicalStrokeCount[] items;
+
+		public UnicodeRadicalStrokeCountCollection() { items = UnicodeRadicalStrokeCount.EmptyArray; }
+		internal UnicodeRadicalStrokeCountCollection(UnicodeRadicalStrokeCount[] items) { this.items = items ?? UnicodeRadicalStrokeCount.EmptyArray; }
+
+		public UnicodeRadicalStrokeCount this[int index] { get { return items[index]; } }
+
+		UnicodeRadicalStrokeCount IList<UnicodeRadicalStrokeCount>.this[int index]
+		{
+			get { return items[index]; }
+			set { throw new NotSupportedException(); }
+		}
+
+		public int Count { get { return items.Length; } }
+
+		bool ICollection<UnicodeRadicalStrokeCount>.IsReadOnly { get { return true; } }
+
+		public void Add(UnicodeRadicalStrokeCount item) { throw new NotSupportedException(); }
+		public void Insert(int index, UnicodeRadicalStrokeCount item) { throw new NotSupportedException(); }
+
+		public bool Remove(UnicodeRadicalStrokeCount item) { throw new NotSupportedException(); }
+		public void RemoveAt(int index) { throw new NotSupportedException(); }
+
+		public void Clear() { throw new NotSupportedException(); }
+
+		public int IndexOf(UnicodeRadicalStrokeCount item) { return Array.IndexOf(items, item); }
+		public bool Contains(UnicodeRadicalStrokeCount item) { return IndexOf(item) >= 0; }
+
+		public void CopyTo(UnicodeRadicalStrokeCount[] array, int arrayIndex) { items.CopyTo(array, arrayIndex); }
+
+		public Enumerator GetEnumerator() { return new Enumerator(items); }
+
+		IEnumerator<UnicodeRadicalStrokeCount> IEnumerable<UnicodeRadicalStrokeCount>.GetEnumerator() { return GetEnumerator(); }
+		IEnumerator IEnumerable.GetEnumerator() { return GetEnumerator(); }
+	}
+}
diff --git a/UnicodeInformation/UnihanCharacterData.cs b/UnicodeInformation/UnihanCharacterData.cs
@@ -11,6 +11,7 @@ internal sealed class UnihanCharacterData
 		public readonly int CodePoint;
 		public readonly UnihanNumericType NumericType;
 		public readonly long NumericValue;
+		public readonly UnicodeRadicalStrokeCount[] UnicodeRadicalStrokeCounts;
 		public readonly string Definition;
 		public readonly string MandarinReading;
 		public readonly string CantoneseReading;
@@ -27,6 +28,7 @@ internal UnihanCharacterData
 			int codePoint,
 			UnihanNumericType numericType,
 			long numericValue,
+			UnicodeRadicalStrokeCount[] unicodeRadicalStrokeCounts,
 			string definition,
 			string mandarinReading,
 			string cantoneseReading,
@@ -42,6 +44,7 @@ string traditionalVariant
 			CodePoint = codePoint;
 			NumericType = numericType;
 			NumericValue = numericValue;
+			UnicodeRadicalStrokeCounts = unicodeRadicalStrokeCounts;
 			Definition = definition;
 			MandarinReading = mandarinReading;
 			CantoneseReading = cantoneseReading;
diff --git a/UnicodeInformation/UnihanFields.cs b/UnicodeInformation/UnihanFields.cs
@@ -14,16 +14,21 @@ internal enum UnihanFields : ushort
 		AccountingNumeric = 2,
 		OtherNumeric = 3,
 
-		Definition = 4,
-		MandarinReading = 8,
-		CantoneseReading = 16,
-		JapaneseKunReading = 32,
-		JapaneseOnReading = 64,
-		KoreanReading = 128,
-		HangulReading = 256,
-		VietnameseReading = 512,
+		// UnicodeRadicalStroke : Not exactly a bit mask…
+		UnicodeRadicalStrokeCount = 4, // Will indicate exactly one value for Unicode_Radical_Stroke.
+		UnicodeRadicalStrokeCountTwice = 8,	// Will indicate exactly two values for Unicode_Radical_Stroke.
+		UnicodeRadicalStrokeCountMore = 12,	// Will indicate three or more values for Unicode_Radical_Stroke. This combination should never happen in the current files.
 
-		SimplifiedVariant = 1024,
-		TraditionalVariant = 2048,
+		Definition = 16,
+		MandarinReading = 32,
+		CantoneseReading = 64,
+		JapaneseKunReading = 128,
+		JapaneseOnReading = 256,
+		KoreanReading = 512,
+		HangulReading = 1024,
+		VietnameseReading = 2048,
+
+		SimplifiedVariant = 4096,
+		TraditionalVariant = 8192,
 	}
 }
diff --git a/UnicodeInformation/ucd.dat b/UnicodeInformation/ucd.dat
