From a9054c6cd97ff9895df74d016aee329e4db7422b Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 5 Nov 2025 18:30:47 +0000 Subject: [PATCH 01/10] Initial plan From 5e362c1b1454921bdb0b7f7701c5a8226de2c1d1 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 5 Nov 2025 18:41:50 +0000 Subject: [PATCH 02/10] Optimize MongoDB attribute loading with explicit batch size for large groups Co-authored-by: simpleidserver <10213388+simpleidserver@users.noreply.github.com> --- .../Models/SCIMRepresentationModel.cs | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/src/Scim/SimpleIdServer.Scim.Persistence.MongoDB/Models/SCIMRepresentationModel.cs b/src/Scim/SimpleIdServer.Scim.Persistence.MongoDB/Models/SCIMRepresentationModel.cs index c122b8a4d..b80e5cfaa 100644 --- a/src/Scim/SimpleIdServer.Scim.Persistence.MongoDB/Models/SCIMRepresentationModel.cs +++ b/src/Scim/SimpleIdServer.Scim.Persistence.MongoDB/Models/SCIMRepresentationModel.cs @@ -46,9 +46,22 @@ public void IncludeSchemas(IMongoDatabase database) public async Task IncludeAttributes(SCIMDbContext dbContext) { - FlatAttributes = await dbContext.SCIMRepresentationAttributeLst.AsQueryable() - .Where(a => a.RepresentationId == Id) - .ToMongoListAsync(); + // Optimize MongoDB query for large result sets by using Find with explicit options + // This provides better performance than LINQ for large collections (e.g., groups with 20k+ members) + var filter = Builders.Filter.Eq(a => a.RepresentationId, Id); + var findOptions = new FindOptions + { + // Use larger batch size to reduce round trips to MongoDB + // Default first batch is 101, then 16MB chunks - we optimize for large groups + BatchSize = 10000, // Fetch up to 10k documents per batch + // Skip building intermediate LINQ expression trees + NoCursorTimeout = false + }; + + var cursor = await dbContext.SCIMRepresentationAttributeLst + .FindAsync(filter, findOptions); + + FlatAttributes = await cursor.ToListAsync(); } } } From e109ef05a0c00fda524da15ef9bc3b76a7bcf22d Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 5 Nov 2025 18:42:52 +0000 Subject: [PATCH 03/10] Optimize FindSCIMRepresentations attribute loading with batch size Co-authored-by: simpleidserver <10213388+simpleidserver@users.noreply.github.com> --- .../SCIMRepresentationQueryRepository.cs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/Scim/SimpleIdServer.Scim.Persistence.MongoDB/SCIMRepresentationQueryRepository.cs b/src/Scim/SimpleIdServer.Scim.Persistence.MongoDB/SCIMRepresentationQueryRepository.cs index f97506b36..d7d7d8354 100644 --- a/src/Scim/SimpleIdServer.Scim.Persistence.MongoDB/SCIMRepresentationQueryRepository.cs +++ b/src/Scim/SimpleIdServer.Scim.Persistence.MongoDB/SCIMRepresentationQueryRepository.cs @@ -97,9 +97,15 @@ join b in _scimDbContext.SCIMRepresentationAttributeLst.AsQueryable() on a.Paren var representationsList = await filteredRepresentations.ToListAsync(cancellationToken); var representationIds = representationsList.Select(r => r.Id).ToList(); - var attributes = await _scimDbContext.SCIMRepresentationAttributeLst.AsQueryable() - .Where(a => representationIds.Contains(a.RepresentationId)) - .ToListAsync(cancellationToken); + // Optimize MongoDB query for potentially large result sets (e.g., when querying groups with many members) + // Use Find API with explicit batch size instead of LINQ for better performance + var filter = Builders.Filter.In(a => a.RepresentationId, representationIds); + var findOptions = new FindOptions + { + BatchSize = 10000 // Fetch up to 10k documents per batch to reduce round trips + }; + var cursor = await _scimDbContext.SCIMRepresentationAttributeLst.FindAsync(filter, findOptions, cancellationToken); + var attributes = await cursor.ToListAsync(cancellationToken); var attributesByRepId = attributes.GroupBy(a => a.RepresentationId) .ToDictionary(g => g.Key, g => g.ToList()); From 227dbc953c7790ab3e453889b20abbcfba703afb Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 5 Nov 2025 18:47:48 +0000 Subject: [PATCH 04/10] Fix misleading comment about NoCursorTimeout setting Co-authored-by: simpleidserver <10213388+simpleidserver@users.noreply.github.com> --- .../Models/SCIMRepresentationModel.cs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/Scim/SimpleIdServer.Scim.Persistence.MongoDB/Models/SCIMRepresentationModel.cs b/src/Scim/SimpleIdServer.Scim.Persistence.MongoDB/Models/SCIMRepresentationModel.cs index b80e5cfaa..8c504a01c 100644 --- a/src/Scim/SimpleIdServer.Scim.Persistence.MongoDB/Models/SCIMRepresentationModel.cs +++ b/src/Scim/SimpleIdServer.Scim.Persistence.MongoDB/Models/SCIMRepresentationModel.cs @@ -53,9 +53,7 @@ public async Task IncludeAttributes(SCIMDbContext dbContext) { // Use larger batch size to reduce round trips to MongoDB // Default first batch is 101, then 16MB chunks - we optimize for large groups - BatchSize = 10000, // Fetch up to 10k documents per batch - // Skip building intermediate LINQ expression trees - NoCursorTimeout = false + BatchSize = 10000 // Fetch up to 10k documents per batch }; var cursor = await dbContext.SCIMRepresentationAttributeLst From d894f5c6d2eedc238f1aa3820ab5a0df48eeedfc Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 5 Nov 2025 18:50:31 +0000 Subject: [PATCH 05/10] Make MongoDB batch size configurable via MongoDbOptions Co-authored-by: simpleidserver <10213388+simpleidserver@users.noreply.github.com> --- .../Models/SCIMRepresentationModel.cs | 5 ++--- .../MongoDbOptions.cs | 7 +++++++ .../SCIMRepresentationQueryRepository.cs | 2 +- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/Scim/SimpleIdServer.Scim.Persistence.MongoDB/Models/SCIMRepresentationModel.cs b/src/Scim/SimpleIdServer.Scim.Persistence.MongoDB/Models/SCIMRepresentationModel.cs index 8c504a01c..4d3440c25 100644 --- a/src/Scim/SimpleIdServer.Scim.Persistence.MongoDB/Models/SCIMRepresentationModel.cs +++ b/src/Scim/SimpleIdServer.Scim.Persistence.MongoDB/Models/SCIMRepresentationModel.cs @@ -51,9 +51,8 @@ public async Task IncludeAttributes(SCIMDbContext dbContext) var filter = Builders.Filter.Eq(a => a.RepresentationId, Id); var findOptions = new FindOptions { - // Use larger batch size to reduce round trips to MongoDB - // Default first batch is 101, then 16MB chunks - we optimize for large groups - BatchSize = 10000 // Fetch up to 10k documents per batch + // Use configured batch size to reduce round trips to MongoDB + BatchSize = dbContext.Options.BatchSize }; var cursor = await dbContext.SCIMRepresentationAttributeLst diff --git a/src/Scim/SimpleIdServer.Scim.Persistence.MongoDB/MongoDbOptions.cs b/src/Scim/SimpleIdServer.Scim.Persistence.MongoDB/MongoDbOptions.cs index 22d48571a..d414e1594 100644 --- a/src/Scim/SimpleIdServer.Scim.Persistence.MongoDB/MongoDbOptions.cs +++ b/src/Scim/SimpleIdServer.Scim.Persistence.MongoDB/MongoDbOptions.cs @@ -16,6 +16,7 @@ public MongoDbOptions() CollectionProvisioningLst = "provisioningLst"; CollectionRealms = "realms"; SupportTransaction = true; + BatchSize = 10000; } public string ConnectionString { get; set; } @@ -28,5 +29,11 @@ public MongoDbOptions() public string CollectionRealms { get; set; } public bool SupportTransaction { get; set; } + + /// + /// MongoDB cursor batch size for large result sets. + /// Default is 10000 to optimize performance when loading groups with many members. + /// + public int BatchSize { get; set; } } } diff --git a/src/Scim/SimpleIdServer.Scim.Persistence.MongoDB/SCIMRepresentationQueryRepository.cs b/src/Scim/SimpleIdServer.Scim.Persistence.MongoDB/SCIMRepresentationQueryRepository.cs index d7d7d8354..ec10eb389 100644 --- a/src/Scim/SimpleIdServer.Scim.Persistence.MongoDB/SCIMRepresentationQueryRepository.cs +++ b/src/Scim/SimpleIdServer.Scim.Persistence.MongoDB/SCIMRepresentationQueryRepository.cs @@ -102,7 +102,7 @@ join b in _scimDbContext.SCIMRepresentationAttributeLst.AsQueryable() on a.Paren var filter = Builders.Filter.In(a => a.RepresentationId, representationIds); var findOptions = new FindOptions { - BatchSize = 10000 // Fetch up to 10k documents per batch to reduce round trips + BatchSize = _scimDbContext.Options.BatchSize }; var cursor = await _scimDbContext.SCIMRepresentationAttributeLst.FindAsync(filter, findOptions, cancellationToken); var attributes = await cursor.ToListAsync(cancellationToken); From 81c4f5228105246c21eade0a987e637e33d3bc88 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 5 Nov 2025 18:54:42 +0000 Subject: [PATCH 06/10] Add cancellation token support to IncludeAll and IncludeAttributes Co-authored-by: simpleidserver <10213388+simpleidserver@users.noreply.github.com> --- .../Models/SCIMRepresentationModel.cs | 11 ++++++----- .../SCIMRepresentationQueryRepository.cs | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/Scim/SimpleIdServer.Scim.Persistence.MongoDB/Models/SCIMRepresentationModel.cs b/src/Scim/SimpleIdServer.Scim.Persistence.MongoDB/Models/SCIMRepresentationModel.cs index 4d3440c25..bc677cf15 100644 --- a/src/Scim/SimpleIdServer.Scim.Persistence.MongoDB/Models/SCIMRepresentationModel.cs +++ b/src/Scim/SimpleIdServer.Scim.Persistence.MongoDB/Models/SCIMRepresentationModel.cs @@ -6,6 +6,7 @@ using SimpleIdServer.Scim.Persistence.MongoDB.Infrastructures; using System.Collections.Generic; using System.Linq; +using System.Threading; using System.Threading.Tasks; namespace SimpleIdServer.Scim.Persistence.MongoDB.Models @@ -33,10 +34,10 @@ public SCIMRepresentationModel(SCIMRepresentation representation, string schemaC public ICollection SchemaRefs { get; set; } public ICollection AttributeRefs { get; set; } - public async Task IncludeAll(SCIMDbContext dbContext) + public async Task IncludeAll(SCIMDbContext dbContext, CancellationToken cancellationToken = default) { IncludeSchemas(dbContext.Database); - await IncludeAttributes(dbContext); + await IncludeAttributes(dbContext, cancellationToken); } public void IncludeSchemas(IMongoDatabase database) @@ -44,7 +45,7 @@ public void IncludeSchemas(IMongoDatabase database) Schemas = MongoDBEntity.GetReferences(SchemaRefs, database); } - public async Task IncludeAttributes(SCIMDbContext dbContext) + public async Task IncludeAttributes(SCIMDbContext dbContext, CancellationToken cancellationToken = default) { // Optimize MongoDB query for large result sets by using Find with explicit options // This provides better performance than LINQ for large collections (e.g., groups with 20k+ members) @@ -56,9 +57,9 @@ public async Task IncludeAttributes(SCIMDbContext dbContext) }; var cursor = await dbContext.SCIMRepresentationAttributeLst - .FindAsync(filter, findOptions); + .FindAsync(filter, findOptions, cancellationToken); - FlatAttributes = await cursor.ToListAsync(); + FlatAttributes = await cursor.ToListAsync(cancellationToken); } } } diff --git a/src/Scim/SimpleIdServer.Scim.Persistence.MongoDB/SCIMRepresentationQueryRepository.cs b/src/Scim/SimpleIdServer.Scim.Persistence.MongoDB/SCIMRepresentationQueryRepository.cs index ec10eb389..1dd12ab62 100644 --- a/src/Scim/SimpleIdServer.Scim.Persistence.MongoDB/SCIMRepresentationQueryRepository.cs +++ b/src/Scim/SimpleIdServer.Scim.Persistence.MongoDB/SCIMRepresentationQueryRepository.cs @@ -153,7 +153,7 @@ public async Task FindSCIMRepresentationById(string realm, s return null; } - await result.IncludeAll(_scimDbContext); + await result.IncludeAll(_scimDbContext, cancellationToken); return result; } From 7fb7dfb21f13406a988dec10d857dc10b6bb1fb8 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 5 Nov 2025 18:56:12 +0000 Subject: [PATCH 07/10] Clarify comment about attribute loading optimization Co-authored-by: simpleidserver <10213388+simpleidserver@users.noreply.github.com> --- .../Models/SCIMRepresentationModel.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Scim/SimpleIdServer.Scim.Persistence.MongoDB/Models/SCIMRepresentationModel.cs b/src/Scim/SimpleIdServer.Scim.Persistence.MongoDB/Models/SCIMRepresentationModel.cs index bc677cf15..251b2c200 100644 --- a/src/Scim/SimpleIdServer.Scim.Persistence.MongoDB/Models/SCIMRepresentationModel.cs +++ b/src/Scim/SimpleIdServer.Scim.Persistence.MongoDB/Models/SCIMRepresentationModel.cs @@ -48,7 +48,7 @@ public void IncludeSchemas(IMongoDatabase database) public async Task IncludeAttributes(SCIMDbContext dbContext, CancellationToken cancellationToken = default) { // Optimize MongoDB query for large result sets by using Find with explicit options - // This provides better performance than LINQ for large collections (e.g., groups with 20k+ members) + // This provides better performance than LINQ for representations with many attributes (e.g., a group with 20k+ member attributes) var filter = Builders.Filter.Eq(a => a.RepresentationId, Id); var findOptions = new FindOptions { From 3616588b71bae992c1345df471222dba96568973 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 5 Nov 2025 18:59:00 +0000 Subject: [PATCH 08/10] Add explicit cursor disposal and BatchSize validation Co-authored-by: simpleidserver <10213388+simpleidserver@users.noreply.github.com> --- .../Models/SCIMRepresentationModel.cs | 2 +- .../MongoDbOptions.cs | 8 +++++++- .../SCIMRepresentationQueryRepository.cs | 2 +- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/Scim/SimpleIdServer.Scim.Persistence.MongoDB/Models/SCIMRepresentationModel.cs b/src/Scim/SimpleIdServer.Scim.Persistence.MongoDB/Models/SCIMRepresentationModel.cs index 251b2c200..9d1f57b31 100644 --- a/src/Scim/SimpleIdServer.Scim.Persistence.MongoDB/Models/SCIMRepresentationModel.cs +++ b/src/Scim/SimpleIdServer.Scim.Persistence.MongoDB/Models/SCIMRepresentationModel.cs @@ -56,7 +56,7 @@ public async Task IncludeAttributes(SCIMDbContext dbContext, CancellationToken c BatchSize = dbContext.Options.BatchSize }; - var cursor = await dbContext.SCIMRepresentationAttributeLst + using var cursor = await dbContext.SCIMRepresentationAttributeLst .FindAsync(filter, findOptions, cancellationToken); FlatAttributes = await cursor.ToListAsync(cancellationToken); diff --git a/src/Scim/SimpleIdServer.Scim.Persistence.MongoDB/MongoDbOptions.cs b/src/Scim/SimpleIdServer.Scim.Persistence.MongoDB/MongoDbOptions.cs index d414e1594..30c118f64 100644 --- a/src/Scim/SimpleIdServer.Scim.Persistence.MongoDB/MongoDbOptions.cs +++ b/src/Scim/SimpleIdServer.Scim.Persistence.MongoDB/MongoDbOptions.cs @@ -30,10 +30,16 @@ public MongoDbOptions() public bool SupportTransaction { get; set; } + private int _batchSize = 10000; /// /// MongoDB cursor batch size for large result sets. /// Default is 10000 to optimize performance when loading groups with many members. + /// Minimum value is 1, maximum recommended is 16MB worth of documents. /// - public int BatchSize { get; set; } + public int BatchSize + { + get => _batchSize; + set => _batchSize = value > 0 ? value : throw new System.ArgumentException("BatchSize must be greater than 0", nameof(BatchSize)); + } } } diff --git a/src/Scim/SimpleIdServer.Scim.Persistence.MongoDB/SCIMRepresentationQueryRepository.cs b/src/Scim/SimpleIdServer.Scim.Persistence.MongoDB/SCIMRepresentationQueryRepository.cs index 1dd12ab62..4bf0c233a 100644 --- a/src/Scim/SimpleIdServer.Scim.Persistence.MongoDB/SCIMRepresentationQueryRepository.cs +++ b/src/Scim/SimpleIdServer.Scim.Persistence.MongoDB/SCIMRepresentationQueryRepository.cs @@ -104,7 +104,7 @@ join b in _scimDbContext.SCIMRepresentationAttributeLst.AsQueryable() on a.Paren { BatchSize = _scimDbContext.Options.BatchSize }; - var cursor = await _scimDbContext.SCIMRepresentationAttributeLst.FindAsync(filter, findOptions, cancellationToken); + using var cursor = await _scimDbContext.SCIMRepresentationAttributeLst.FindAsync(filter, findOptions, cancellationToken); var attributes = await cursor.ToListAsync(cancellationToken); var attributesByRepId = attributes.GroupBy(a => a.RepresentationId) From 1b3ea35663b96ff7c6403513a54cd29f990322a2 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 5 Nov 2025 19:01:36 +0000 Subject: [PATCH 09/10] Fix BatchSize validation to use ArgumentOutOfRangeException and proper imports Co-authored-by: simpleidserver <10213388+simpleidserver@users.noreply.github.com> --- .../MongoDbOptions.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Scim/SimpleIdServer.Scim.Persistence.MongoDB/MongoDbOptions.cs b/src/Scim/SimpleIdServer.Scim.Persistence.MongoDB/MongoDbOptions.cs index 30c118f64..d3fefb2b1 100644 --- a/src/Scim/SimpleIdServer.Scim.Persistence.MongoDB/MongoDbOptions.cs +++ b/src/Scim/SimpleIdServer.Scim.Persistence.MongoDB/MongoDbOptions.cs @@ -1,5 +1,6 @@ // Copyright (c) SimpleIdServer. All rights reserved. // Licensed under the Apache License, Version 2.0. See LICENSE in the project root for license information. +using System; namespace SimpleIdServer.Scim.Persistence.MongoDB { @@ -30,7 +31,7 @@ public MongoDbOptions() public bool SupportTransaction { get; set; } - private int _batchSize = 10000; + private int _batchSize; /// /// MongoDB cursor batch size for large result sets. /// Default is 10000 to optimize performance when loading groups with many members. @@ -39,7 +40,7 @@ public MongoDbOptions() public int BatchSize { get => _batchSize; - set => _batchSize = value > 0 ? value : throw new System.ArgumentException("BatchSize must be greater than 0", nameof(BatchSize)); + set => _batchSize = value > 0 ? value : throw new ArgumentOutOfRangeException(nameof(BatchSize), "BatchSize must be greater than 0"); } } } From 507f3664a7cd4373abc9d8a6129916e6e7d86477 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 5 Nov 2025 19:04:50 +0000 Subject: [PATCH 10/10] Improve BatchSize property setter readability and documentation Co-authored-by: simpleidserver <10213388+simpleidserver@users.noreply.github.com> --- .../MongoDbOptions.cs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/Scim/SimpleIdServer.Scim.Persistence.MongoDB/MongoDbOptions.cs b/src/Scim/SimpleIdServer.Scim.Persistence.MongoDB/MongoDbOptions.cs index d3fefb2b1..3ccea108b 100644 --- a/src/Scim/SimpleIdServer.Scim.Persistence.MongoDB/MongoDbOptions.cs +++ b/src/Scim/SimpleIdServer.Scim.Persistence.MongoDB/MongoDbOptions.cs @@ -35,12 +35,18 @@ public MongoDbOptions() /// /// MongoDB cursor batch size for large result sets. /// Default is 10000 to optimize performance when loading groups with many members. - /// Minimum value is 1, maximum recommended is 16MB worth of documents. + /// Higher values reduce network round trips but increase memory usage. + /// Minimum value is 1. /// public int BatchSize { get => _batchSize; - set => _batchSize = value > 0 ? value : throw new ArgumentOutOfRangeException(nameof(BatchSize), "BatchSize must be greater than 0"); + set + { + if (value <= 0) + throw new ArgumentOutOfRangeException(nameof(BatchSize), "BatchSize must be greater than 0"); + _batchSize = value; + } } } }