Merge duplicate analyses (#282)

* Add MergeEquivalentAnalyses

* Fix a performance bug

* Optimization should include when MergeEquivalentAnalyses is false

* Uses TryGetWord as suggested by Damien

Author: jtmaxwell3

src/SIL.Machine.Morphology.HermitCrab/AnalysisStratumRule.cs

@@ -67,11 +67,17 @@ public IEnumerable<Word> Apply(Word input)
             if (_morpher.TraceManager.IsTracing)
                 _morpher.TraceManager.BeginUnapplyStratum(_stratum, input);
 
+            Word origInput = input;
             input = input.Clone();
             input.Stratum = _stratum;
 
             _prulesRule.Apply(input);
             input.Freeze();
+            IDictionary<Shape, Word> shapeWord = null;
+            // Don't merge if tracing because it messes up the tracing.
+            bool mergeEquivalentAnalyses = _morpher.MergeEquivalentAnalyses && !_morpher.TraceManager.IsTracing;
+            if (mergeEquivalentAnalyses)
+                shapeWord = new Dictionary<Shape, Word>(FreezableEqualityComparer<Shape>.Default);
 
             // AnalysisStratumRule.Apply should cover the inverse of SynthesisStratumRule.Apply.
             IEnumerable<Word> mruleOutWords = ApplyTemplates(input).Concat(ApplyMorphologicalRules(input));
@@ -82,6 +88,19 @@ public IEnumerable<Word> Apply(Word input)
                 _morpher.TraceManager.EndUnapplyStratum(_stratum, input);
             foreach (Word mruleOutWord in mruleOutWords)
             {
+                // Skip intermediate sources from phonological rules, templates, and morphological rules.
+                mruleOutWord.Source = origInput;
+                if (mergeEquivalentAnalyses)
+                {
+                    Shape shape = mruleOutWord.Shape;
+                    Word canonicalWord;
+                    if (shapeWord.TryGetValue(shape, out canonicalWord))
+                    {
+                        canonicalWord.Alternatives.Add(mruleOutWord);
+                        continue;
+                    }
+                    shapeWord[shape] = mruleOutWord;
+                }
                 output.Add(mruleOutWord);
                 if (_morpher.TraceManager.IsTracing)
                     _morpher.TraceManager.EndUnapplyStratum(_stratum, mruleOutWord);

src/SIL.Machine.Morphology.HermitCrab/Morpher.cs

@@ -56,6 +56,7 @@ public Morpher(ITraceManager traceManager, Language lang)
             _synthesisRule = lang.CompileSynthesisRule(this);
             MaxStemCount = 2;
             MaxUnapplications = 0;
+            MergeEquivalentAnalyses = true;
             LexEntrySelector = entry => true;
             RuleSelector = rule => true;
 
@@ -78,6 +79,12 @@ public ITraceManager TraceManager
         /// </summary>
         public int MaxUnapplications { get; set; }
 
+        /// <summary>
+        /// Merge analyses that have equivalent shapes.
+        /// Merged analyses will be expanded if lexical lookup succeeds.
+        /// </summary>
+        public bool MergeEquivalentAnalyses { get; set; }
+
         public Func<LexEntry, bool> LexEntrySelector { get; set; }
         public Func<IHCRule, bool> RuleSelector { get; set; }
 
@@ -139,10 +146,13 @@ public IEnumerable<Word> ParseWord(string word, out object trace, bool guessRoot
                     var lexicalGuesses = LexicalGuess(analysisWord).Distinct();
                     foreach (Word synthesisWord in lexicalGuesses)
                     {
-                        foreach (Word validWord in _synthesisRule.Apply(synthesisWord).Where(IsWordValid))
+                        foreach (Word alternative in synthesisWord.ExpandAlternatives())
                         {
-                            if (IsMatch(word, validWord))
-                                matches.Add(validWord);
+                            foreach (Word validWord in _synthesisRule.Apply(alternative).Where(IsWordValid))
+                            {
+                                if (IsMatch(word, validWord))
+                                    matches.Add(validWord);
+                            }
                         }
                     }
                 }
@@ -278,10 +288,13 @@ private IEnumerable<Word> Synthesize(string word, IEnumerable<Word> analyses)
             {
                 foreach (Word synthesisWord in LexicalLookup(analysisWord))
                 {
-                    foreach (Word validWord in _synthesisRule.Apply(synthesisWord).Where(IsWordValid))
+                    foreach (Word alternative in synthesisWord.ExpandAlternatives())
                     {
-                        if (IsMatch(word, validWord))
-                            matches.Add(validWord);
+                        foreach (Word validWord in _synthesisRule.Apply(alternative).Where(IsWordValid))
+                        {
+                            if (IsMatch(word, validWord))
+                                matches.Add(validWord);
+                        }
                     }
                 }
             }
@@ -304,10 +317,13 @@ private IEnumerable<Word> Synthesize(string word, ConcurrentQueue<Word> analyses
                             analyses.TryDequeue(out Word analysisWord);
                             foreach (Word synthesisWord in LexicalLookup(analysisWord))
                             {
-                                foreach (Word validWord in _synthesisRule.Apply(synthesisWord).Where(IsWordValid))
+                                foreach (Word alternative in synthesisWord.ExpandAlternatives())
                                 {
-                                    if (IsMatch(word, validWord))
-                                        matches.Add(validWord);
+                                    foreach (Word validWord in _synthesisRule.Apply(alternative).Where(IsWordValid))
+                                    {
+                                        if (IsMatch(word, validWord))
+                                            matches.Add(validWord);
+                                    }
                                 }
                             }
                         }

src/SIL.Machine.Morphology.HermitCrab/Word.cs

@@ -31,6 +31,7 @@ public class Word : Freezable<Word>, IAnnotatedData<ShapeNode>, ICloneable<Word>
         private bool _isPartial;
         private readonly Dictionary<string, HashSet<int>> _disjunctiveAllomorphIndices;
         private int _mruleAppCount = 0;
+        private readonly IList<Word> _alternatives = new List<Word>();
 
         public Word(RootAllomorph rootAllomorph, FeatureStruct realizationalFS)
         {
@@ -72,6 +73,8 @@ protected Word(Word word)
         {
             _allomorphs = new Dictionary<string, Allomorph>(word._allomorphs);
             Stratum = word.Stratum;
+            Source = word;
+            // Don't copy Alternatives.
             _shape = word._shape.Clone();
             _rootAllomorph = word._rootAllomorph;
             SyntacticFeatureStruct = word.SyntacticFeatureStruct.Clone();
@@ -396,6 +399,48 @@ internal void NonHeadUnapplied(Word nonHead)
             _nonHeadAppIndex++;
         }
 
+        internal Word Source { get; set; }
+
+        internal IList<Word> Alternatives
+        {
+            get { return _alternatives; }
+        }
+
+        internal IList<Word> ExpandAlternatives()
+        {
+            IList<Word> alternatives = new List<Word>();
+            IList<Word> originals = Source?.ExpandAlternatives();
+            // Update the alternatives of CloneOf with any changes made since the clone.
+            if (originals == null || originals.Count < 2)
+            {
+                // Special case.
+                alternatives.Add(this);
+            }
+            else
+            {
+                foreach (Word original in originals)
+                {
+                    Word alternative = original.Clone();
+                    alternative._shape = this.Shape;
+                    // Add new rules to alternative.
+                    int m_start = Source == null ? 0 : Source._mruleApps.Count();
+                    for (int i = m_start; i < _mruleApps.Count(); i++)
+                        alternative.MorphologicalRuleUnapplied(_mruleApps[i]);
+                    int nh_start = Source == null ? 0 : Source._nonHeadApps.Count();
+                    for (int i = nh_start; i < _nonHeadApps.Count(); i++)
+                        alternative.NonHeadUnapplied(_nonHeadApps[i]);
+                    if (RootAllomorph != null)
+                        alternative.RootAllomorph = RootAllomorph;
+                    alternative.Freeze();
+                    alternatives.Add(alternative);
+                }
+            }
+            // Add local alternatives.
+            foreach (Word alternative in _alternatives)
+                alternatives.AddRange(alternative.ExpandAlternatives());
+            return alternatives;
+        }
+
         public Allomorph GetAllomorph(Annotation<ShapeNode> morph)
         {
             var alloID = (string)morph.FeatureStruct.GetValue(HCFeatureSystem.Allomorph);