Catch up code to upstream 0.19 release (d9439d6)

Author: paulirwin

src/F23.StringSimilarity/Cosine.cs

@@ -23,7 +23,9 @@
  */
 
 using System;
+using System.Collections.Generic;
 using F23.StringSimilarity.Interfaces;
+// ReSharper disable LoopCanBeConvertedToQuery
 
 namespace F23.StringSimilarity
 {
@@ -47,17 +49,16 @@ public Cosine(int k) : base(k) { }
         /// Default k is 3.
         /// </summary>
         public Cosine() { }
-        
+
         public double Similarity(string s1, string s2)
         {
             if (s1.Length < k || s2.Length < k)
             {
                 return 0;
             }
 
-            KShingling ks = new KShingling(k);
-            int[] profile1 = ks.GetArrayProfile(s1);
-            int[] profile2 = ks.GetArrayProfile(s2);
+            var profile1 = GetProfile(s1);
+            var profile2 = GetProfile(s2);
 
             return DotProduct(profile1, profile2) / (Norm(profile1) * Norm(profile2));
         }
@@ -68,31 +69,47 @@ public double Similarity(string s1, string s2)
          * @param profile
          * @return L2 norm
          */
-        private static double Norm(int[] profile)
+        private static double Norm(IDictionary<string, int> profile)
         {
             double agg = 0;
 
-            foreach (var v in profile)
+            foreach (var entry in profile)
             {
-                agg += (double)v * v;
+                agg += 1.0 * entry.Value * entry.Value;
             }
 
             return Math.Sqrt(agg);
         }
 
-        private static double DotProduct(int[] profile1, int[] profile2)
+        private static double DotProduct(IDictionary<string, int> profile1,
+            IDictionary<string, int> profile2)
         {
-            int length = Math.Min(profile1.Length, profile2.Length);
+            // Loop over the smallest map
+            var small_profile = profile2;
+            var large_profile = profile1;
+
+            if (profile1.Count < profile2.Count)
+            {
+                small_profile = profile1;
+                large_profile = profile2;
+            }
 
             double agg = 0;
-            for (int i = 0; i < length; i++)
+            foreach (var entry in small_profile)
             {
-                agg += (double)profile1[i] * profile2[i];
+                if (!large_profile.ContainsKey(entry.Key)) continue;
+
+                agg += 1.0 * entry.Value * large_profile[entry.Key];
             }
+
             return agg;
         }
 
         public double Distance(string s1, string s2)
             => 1.0 - Similarity(s1, s2);
+
+        public double Similarity(IDictionary<string, int> profile1, IDictionary<string, int> profile2)
+            => DotProduct(profile1, profile2)
+            / (Norm(profile1) * Norm(profile2));
     }
 }

src/F23.StringSimilarity/F23.StringSimilarity.csproj

@@ -50,7 +50,6 @@
     <Compile Include="Interfaces\IStringSimilarity.cs" />
     <Compile Include="Jaccard.cs" />
     <Compile Include="JaroWinkler.cs" />
-    <Compile Include="KShingling.cs" />
     <Compile Include="Levenshtein.cs" />
     <Compile Include="LongestCommonSubsequence.cs" />
     <Compile Include="MetricLCS.cs" />
@@ -61,8 +60,6 @@
     <Compile Include="QGram.cs" />
     <Compile Include="ShingleBased.cs" />
     <Compile Include="SorensenDice.cs" />
-    <Compile Include="StringProfile.cs" />
-    <Compile Include="StringSet.cs" />
     <Compile Include="Support\ArrayExtensions.cs" />
     <Compile Include="Utils\SparseBooleanVector.cs" />
     <Compile Include="Utils\SparseIntegerVector.cs" />

src/F23.StringSimilarity/Jaccard.cs

@@ -23,8 +23,10 @@
  */
 
 using System;
+using System.Collections.Generic;
 using F23.StringSimilarity.Interfaces;
 using F23.StringSimilarity.Support;
+// ReSharper disable LoopCanBeConvertedToQuery
 
 namespace F23.StringSimilarity
 {
@@ -42,32 +44,22 @@ public Jaccard() { }
         /// <returns>Similarity</returns>
         public double Similarity(string s1, string s2)
         {
-            KShingling ks = new KShingling(k);
-            int[] profile1 = ks.GetArrayProfile(s1);
-            int[] profile2 = ks.GetArrayProfile(s2);
+            var profile1 = GetProfile(s1);
+            var profile2 = GetProfile(s2);
 
-            int length = Math.Max(profile1.Length, profile2.Length);
-
-            profile1 = profile1.WithPadding(length);
-            profile2 = profile2.WithPadding(length);
+            var union = new HashSet<string>();
+            union.UnionWith(profile1.Keys);
+            union.UnionWith(profile2.Keys);
 
             int inter = 0;
-            int union = 0;
 
-            for (int i = 0; i < length; i++)
+            foreach (var key in union)
             {
-                if (profile1[i] > 0 || profile2[i] > 0)
-                {
-                    union++;
-
-                    if (profile1[i] > 0 && profile2[i] > 0)
-                    {
-                        inter++;
-                    }
-                }
+                if (profile1.ContainsKey(key) && profile2.ContainsKey(key))
+                    inter++;
             }
 
-            return 1.0 * inter / union;
+            return 1.0 * inter / union.Count;
         }
 
 

src/F23.StringSimilarity/KShingling.cs

@@ -60,11 +60,11 @@ public double Distance(string s0, string s1)
             {
                 if (sl == tl)
                 {
-                    return 1;
+                    return 0;
                 }
                 else
                 {
-                    return 0;
+                    return 1;
                 }
             }
 
@@ -128,7 +128,7 @@ public double Distance(string s0, string s1)
                 }
                 else
                 {
-                    t_j = s1.Substring(j - n, j).ToCharArray();
+                    t_j = s1.Substring(j - n, n).ToCharArray();
                 }
                 d[0] = j;
                 for (i = 1; i <= sl; i++)
@@ -160,7 +160,7 @@ public double Distance(string s0, string s1)
 
             // Our last action in the above loop was to switch d and p, so p now
             // actually has the most recent cost counts
-            return 1.0 - (p[sl] / Math.Max(tl, sl));
+            return p[sl] / Math.Max(tl, sl);
         }
     }
 }