Improve SortedSet ctor performance

- SortedSet's ctor has a loop that's O(D * N), where N is the number of elements being added to the set and D is the number of duplicates in those elements.  As the number of duplicates D grows, the time it takes to construct a sorted set grows polynomially, approaching O(N^2), which is particularly bad given that the purpose of a set is to help remove duplicates.  The core cause is a loop over the elements to determine whether element n is equal to n-1; if it is, element n is removed, but it's removed via a call to List<T>.RemoveAt, which will shift down all of the elements above it in the list.  A better implementation can make this O(N) instead; the ctor overall is still then O(N log N), due to the sort, but that's better than O(N^2).
- To sort the elements, the ctor builds a List<T> of the elements but then uses ToArray to get an array used to actually build the set; this incurs an unnecessary (and potentially large) array allocation and copy.  We can avoid that entirely by just building up the array manually using the same logic that List<T> does.  That same logic is actually duplicated in multiple places, including in Stack<T>, so I've extracted it as a helper to use in both places (there are a few other places in corefx that use the same logic, hence I put the helper into a Common file, but I did not condense those other uses as part of this commit).  This then further helps with Stack<T> perf in a few corner cases, such as when a Stack<T> is constructed with an empty enumerable. (It's not clear to me why Queue<T> doesn't have the same IColection<T>-related logic; probably just an oversight, but I didn't want to change its behavior with the additional ICollection<T> check.)

Author: stephentoub

src/Common/src/System/Collections/Generic/EnumerableHelpers.cs

@@ -0,0 +1,85 @@
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+namespace System.Collections.Generic
+{
+    /// <summary>Internal helper functions for working with enumerables.</summary>
+    internal static class EnumerableHelpers
+    {
+        /// <summary>Converts an enumerable to an array using the same logic as does List{T}.</summary>
+        /// <param name="length">The number of items stored in the resulting array, 0-indexed.</param>
+        /// <returns>
+        /// The resulting array.  The length of the array may be greater than <paramref name="length"/>,
+        /// which is the actual number of elements in the array.
+        /// </returns>
+        internal static T[] ToArray<T>(IEnumerable<T> source, out int length)
+        {
+            T[] arr;
+            int count = 0;
+
+            ICollection<T> ic = source as ICollection<T>;
+            if (ic != null)
+            {
+                count = ic.Count;
+                if (count == 0)
+                {
+                    arr = Array.Empty<T>();
+                }
+                else
+                {
+                    // Allocate an array of the desired size, then copy the elements into it. Note that this has the same 
+                    // issue regarding concurrency as other existing collections like List<T>. If the collection size 
+                    // concurrently changes between the array allocation and the CopyTo, we could end up either getting an 
+                    // exception from overrunning the array (if the size went up) or we could end up not filling as many 
+                    // items as 'count' suggests (if the size went down).  This is only an issue for concurrent collections 
+                    // that implement ICollection<T>, which as of .NET 4.6 is just ConcurrentDictionary<TKey, TValue>.
+                    arr = new T[count];
+                    ic.CopyTo(arr, 0);
+                }
+            }
+            else
+            {
+                arr = Array.Empty<T>();
+                foreach (var item in source)
+                {
+                    if (count == arr.Length)
+                    {
+                        // MaxArrayLength is defined in Array.MaxArrayLength and in gchelpers in CoreCLR.
+                        // It represents the maximum number of elements that can be in an array where
+                        // the size of the element is greater than one byte; a separate, slightly larger constant,
+                        // is used when the size of the element is one.
+                        const int MaxArrayLength = 0x7FEFFFFF;
+
+                        // This is the same growth logic as in List<T>:
+                        // If the array is currently empty, we make it a default size.  Otherwise, we attempt to 
+                        // double the size of the array.  Doubling will overflow once the size of the array reaches
+                        // 2^30, since doubling to 2^31 is 1 larger than Int32.MaxValue.  In that case, we instead 
+                        // constrain the length to be MaxArrayLength (this overflow check works because of of the 
+                        // cast to uint).  Because a slightly larger constant is used when T is one byte in size, we 
+                        // could then end up in a situation where arr.Length is MaxArrayLength or slightly larger, such 
+                        // that we constrain newLength to be MaxArrayLength but the needed number of elements is actually 
+                        // larger than that.  For that case, we then ensure that the newLength is large enough to hold 
+                        // the desired capacity.  This does mean that in the very rare case where we've grown to such a 
+                        // large size, each new element added after MaxArrayLength will end up doing a resize.
+                        const int DefaultCapacity = 4;
+                        int newLength = count == 0 ? DefaultCapacity : count * 2;
+                        if ((uint)newLength > MaxArrayLength)
+                        {
+                            newLength = MaxArrayLength;
+                        }
+                        if (newLength < count + 1)
+                        {
+                            newLength = count + 1;
+                        }
+
+                        arr = ArrayT<T>.Resize(arr, newLength, count);
+                    }
+                    arr[count++] = item;
+                }
+            }
+
+            length = count;
+            return arr;
+        }
+    }
+}

src/System.Collections/src/System.Collections.csproj

@@ -42,11 +42,14 @@
     <Compile Include="System\Collections\StructuralComparisons.cs" />
   </ItemGroup>
   <ItemGroup>
+    <Compile Include="$(CommonPath)\System\ArrayT.cs">
+      <Link>Common\System\ArrayT.cs</Link>
+    </Compile>
     <Compile Include="$(CommonPath)\System\Collections\HashHelpers.cs">
       <Link>Common\System\Collections\HashHelpers.cs</Link>
     </Compile>
-    <Compile Include="$(CommonPath)\System\ArrayT.cs">
-      <Link>Common\System\ArrayT.cs</Link>
+    <Compile Include="$(CommonPath)\System\Collections\Generic\EnumerableHelpers.cs">
+      <Link>Common\System\Collections\Generic\EnumerableHelpers.cs</Link>
     </Compile>
   </ItemGroup>
   <ItemGroup>

src/System.Collections/src/System/Collections/Generic/SortedSet.cs

@@ -95,13 +95,9 @@ public SortedSet(IEnumerable<T> collection, IComparer<T> comparer)
                 //breadth first traversal to recreate nodes
                 if (baseSortedSet.Count == 0)
                 {
-                    _count = 0;
-                    _version = 0;
-                    _root = null;
                     return;
                 }
 
-
                 //pre order way to replicate nodes
                 Stack<Node> theirStack = new Stack<SortedSet<T>.Node>(2 * log2(baseSortedSet.Count) + 2);
                 Stack<Node> myStack = new Stack<SortedSet<T>.Node>(2 * log2(baseSortedSet.Count) + 2);
@@ -138,23 +134,27 @@ public SortedSet(IEnumerable<T> collection, IComparer<T> comparer)
                     }
                 }
                 _count = baseSortedSet._count;
-                _version = 0;
             }
             else
-            {     //As it stands, you're doing an NlogN sort of the collection
-                List<T> els = new List<T>(collection);
-                els.Sort(_comparer);
-                for (int i = 1; i < els.Count; i++)
+            {
+                int count;
+                T[] els = EnumerableHelpers.ToArray(collection, out count);
+                if (count > 0)
                 {
-                    if (comparer.Compare(els[i], els[i - 1]) == 0)
+                    Array.Sort(els, 0, count, _comparer);
+                    int index = 1;
+                    for (int i = 1; i < count; i++)
                     {
-                        els.RemoveAt(i);
-                        i--;
+                        if (comparer.Compare(els[i], els[i - 1]) != 0)
+                        {
+                            els[index++] = els[i];
+                        }
                     }
+                    count = index;
+
+                    _root = ConstructRootFromSortedArray(els, 0, count - 1, null);
+                    _count = count;
                 }
-                _root = ConstructRootFromSortedArray(els.ToArray(), 0, els.Count - 1, null);
-                _count = els.Count;
-                _version = 0;
             }
         }
 

src/System.Collections/src/System/Collections/Generic/Stack.cs

@@ -60,27 +60,7 @@ public Stack(IEnumerable<T> collection)
             if (collection == null)
                 throw new ArgumentNullException("collection");
 
-            ICollection<T> c = collection as ICollection<T>;
-            if (c != null)
-            {
-                int count = c.Count;
-                _array = new T[count];
-                c.CopyTo(_array, 0);
-                _size = count;
-            }
-            else
-            {
-                _size = 0;
-                _array = new T[DefaultCapacity];
-
-                using (IEnumerator<T> en = collection.GetEnumerator())
-                {
-                    while (en.MoveNext())
-                    {
-                        Push(en.Current);
-                    }
-                }
-            }
+            _array = EnumerableHelpers.ToArray(collection, out _size);
         }
 
         /// <include file='doc\Stack.uex' path='docs/doc[@for="Stack.Count"]/*' />