Fix bugs in fetching more than 2000 items (permissions and culture variations)

Author: ronaldbarendse

src/Umbraco.Core/Persistence/Repositories/Implement/ContentTypeRepositoryBase.cs

@@ -1094,14 +1094,20 @@ private void RenormalizeDocumentEditedFlags(IReadOnlyCollection<int> propertyTyp
                 }
             }
 
-            //lookup all matching rows in umbracoDocumentCultureVariation
-            var docCultureVariationsToUpdate = editedLanguageVersions.InGroupsOf(Constants.Sql.MaxParameterCount)
-                .SelectMany(_ => Database.Fetch<DocumentCultureVariationDto>(
-                    Sql().Select<DocumentCultureVariationDto>().From<DocumentCultureVariationDto>()
-                            .WhereIn<DocumentCultureVariationDto>(x => x.LanguageId, editedLanguageVersions.Keys.Select(x => x.langId).ToList())
-                            .WhereIn<DocumentCultureVariationDto>(x => x.NodeId, editedLanguageVersions.Keys.Select(x => x.nodeId))))
-                //convert to dictionary with the same key type
-                .ToDictionary(x => (x.NodeId, (int?)x.LanguageId), x => x);
+            // lookup all matching rows in umbracoDocumentCultureVariation
+            // fetch in batches to account for maximum parameter count (distinct languages can't exceed 2000)
+            var languageIds = editedLanguageVersions.Keys.Select(x => x.langId).Distinct().ToArray();
+            var nodeIds = editedLanguageVersions.Keys.Select(x => x.nodeId).Distinct();
+            var docCultureVariationsToUpdate = nodeIds.InGroupsOf(Constants.Sql.MaxParameterCount - languageIds.Length)
+                .SelectMany(group =>
+                {
+                    var sql = Sql().Select<DocumentCultureVariationDto>().From<DocumentCultureVariationDto>()
+                        .WhereIn<DocumentCultureVariationDto>(x => x.LanguageId, languageIds)
+                        .WhereIn<DocumentCultureVariationDto>(x => x.NodeId, group);
+
+                    return Database.Fetch<DocumentCultureVariationDto>(sql);
+                })
+                .ToDictionary(x => (x.NodeId, (int?)x.LanguageId), x => x); //convert to dictionary with the same key type
 
             var toUpdate = new List<DocumentCultureVariationDto>();
             foreach (var ev in editedLanguageVersions)

src/Umbraco.Core/Persistence/Repositories/Implement/PermissionRepository.cs

@@ -38,44 +38,41 @@ public PermissionRepository(IScopeAccessor scopeAccessor, AppCaches cache, ILogg
         /// <param name="entityIds"></param>
         /// <returns></returns>
         /// <remarks>
-        /// This method will not support passing in more than 2000 group Ids
+        /// This method will not support passing in more than 2000 group IDs when also passing in entity IDs.
         /// </remarks>
         public EntityPermissionCollection GetPermissionsForEntities(int[] groupIds, params int[] entityIds)
         {
             var result = new EntityPermissionCollection();
 
-            foreach (var groupOfGroupIds in groupIds.InGroupsOf(Constants.Sql.MaxParameterCount))
+            if (entityIds.Length == 0)
             {
-                //copy local
-                var localIds = groupOfGroupIds.ToArray();
-
-                if (entityIds.Length == 0)
+                foreach (var group in groupIds.InGroupsOf(Constants.Sql.MaxParameterCount))
                 {
                     var sql = Sql()
                         .SelectAll()
                         .From<UserGroup2NodePermissionDto>()
-                        .Where<UserGroup2NodePermissionDto>(dto => localIds.Contains(dto.UserGroupId));
+                        .Where<UserGroup2NodePermissionDto>(dto => group.Contains(dto.UserGroupId));
+
                     var permissions = AmbientScope.Database.Fetch<UserGroup2NodePermissionDto>(sql);
                     foreach (var permission in ConvertToPermissionList(permissions))
                     {
                         result.Add(permission);
                     }
                 }
-                else
+            }
+            else
+            {
+                foreach (var group in entityIds.InGroupsOf(Constants.Sql.MaxParameterCount - groupIds.Length))
                 {
-                    //iterate in groups of 2000 since we don't want to exceed the max SQL param count
-                    foreach (var groupOfEntityIds in entityIds.InGroupsOf(Constants.Sql.MaxParameterCount))
+                    var sql = Sql()
+                        .SelectAll()
+                        .From<UserGroup2NodePermissionDto>()
+                        .Where<UserGroup2NodePermissionDto>(dto => groupIds.Contains(dto.UserGroupId) && group.Contains(dto.NodeId));
+
+                    var permissions = AmbientScope.Database.Fetch<UserGroup2NodePermissionDto>(sql);
+                    foreach (var permission in ConvertToPermissionList(permissions))
                     {
-                        var ids = groupOfEntityIds;
-                        var sql = Sql()
-                            .SelectAll()
-                            .From<UserGroup2NodePermissionDto>()
-                            .Where<UserGroup2NodePermissionDto>(dto => localIds.Contains(dto.UserGroupId) && ids.Contains(dto.NodeId));
-                        var permissions = AmbientScope.Database.Fetch<UserGroup2NodePermissionDto>(sql);
-                        foreach (var permission in ConvertToPermissionList(permissions))
-                        {
-                            result.Add(permission);
-                        }
+                        result.Add(permission);
                     }
                 }
             }