diff --git a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/SqlServerSearchServiceTests.cs b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/SqlServerSearchServiceTests.cs index eb3693d0bb..2c36424f05 100644 --- a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/SqlServerSearchServiceTests.cs +++ b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/SqlServerSearchServiceTests.cs @@ -299,5 +299,108 @@ public void ReuseQueryPlansParameterId_HasExpectedValue() // Assert Assert.Equal("Search.ReuseQueryPlans.IsEnabled", SqlServerSearchService.ReuseQueryPlansParameterId); } + + [Fact] + public void CollectNotExistsLeaves_WithResourceSurrogateId_DetectsSurrogateIdAndMissingParam() + { + // Arrange: NotExists predicate with MissingSearchParameterExpression + ResourceSurrogateId constraint + var genderParam = new SearchParameterInfo("gender", "gender"); + var missingExpr = Expression.MissingSearchParameter(genderParam, false); + var surrogateIdExpr = Expression.SearchParameter( + SqlSearchParameters.ResourceSurrogateIdParameter, + Expression.GreaterThanOrEqual(SqlFieldName.ResourceSurrogateId, null, 100L)); + + var predicate = Expression.And(missingExpr, surrogateIdExpr); + + var missingParams = new List(); + var resourceTypeIds = new HashSet(); + bool foundSurrogateId = false; + + // Act + SqlServerSearchService.ResourceSearchParamStats.CollectNotExistsLeaves( + predicate, missingParams, resourceTypeIds, null, ref foundSurrogateId); + + // Assert + Assert.Single(missingParams); + Assert.Equal("gender", missingParams[0].Parameter.Name); + Assert.True(foundSurrogateId, "Should detect ResourceSurrogateId constraint"); + } + + [Fact] + public void CollectNotExistsLeaves_WithoutResourceSurrogateId_DoesNotSetSurrogateIdFlag() + { + // Arrange: NotExists predicate with only MissingSearchParameterExpression + var genderParam = new SearchParameterInfo("gender", "gender"); + var missingExpr = Expression.MissingSearchParameter(genderParam, false); + + var missingParams = new List(); + var resourceTypeIds = new HashSet(); + bool foundSurrogateId = false; + + // Act + SqlServerSearchService.ResourceSearchParamStats.CollectNotExistsLeaves( + missingExpr, missingParams, resourceTypeIds, null, ref foundSurrogateId); + + // Assert + Assert.Single(missingParams); + Assert.Equal("gender", missingParams[0].Parameter.Name); + Assert.False(foundSurrogateId, "Should not detect ResourceSurrogateId when absent"); + } + + [Fact] + public void CollectNotExistsLeaves_AmbiguousSameTablePredicates_CollectsMultipleMissingParams() + { + // Arrange: Two MissingSearchParameterExpression nodes in the same predicate (ambiguous) + var genderParam = new SearchParameterInfo("gender", "gender"); + var codeParam = new SearchParameterInfo("code", "code"); + var missing1 = Expression.MissingSearchParameter(genderParam, false); + var missing2 = Expression.MissingSearchParameter(codeParam, false); + + var predicate = Expression.And(missing1, missing2); + + var missingParams = new List(); + var resourceTypeIds = new HashSet(); + bool foundSurrogateId = false; + + // Act + SqlServerSearchService.ResourceSearchParamStats.CollectNotExistsLeaves( + predicate, missingParams, resourceTypeIds, null, ref foundSurrogateId); + + // Assert: Two missing params collected — ProcessNotExistsForStats would skip this (ambiguous) + Assert.Equal(2, missingParams.Count); + Assert.False(foundSurrogateId); + } + + [Fact] + public void CollectNotExistsLeaves_MixedPredicates_CollectsAllLeafTypes() + { + // Arrange: MissingSearchParameterExpression + unrelated param + ResourceSurrogateId + var genderParam = new SearchParameterInfo("gender", "gender"); + var missingExpr = Expression.MissingSearchParameter(genderParam, false); + + var otherParam = new SearchParameterInfo("active", "active"); + var otherExpr = Expression.SearchParameter( + otherParam, + Expression.StringEquals(FieldName.TokenCode, null, "true", false)); + + var surrogateIdExpr = Expression.SearchParameter( + SqlSearchParameters.ResourceSurrogateIdParameter, + Expression.GreaterThanOrEqual(SqlFieldName.ResourceSurrogateId, null, 100L)); + + var predicate = Expression.And(missingExpr, otherExpr, surrogateIdExpr); + + var missingParams = new List(); + var resourceTypeIds = new HashSet(); + bool foundSurrogateId = false; + + // Act + SqlServerSearchService.ResourceSearchParamStats.CollectNotExistsLeaves( + predicate, missingParams, resourceTypeIds, null, ref foundSurrogateId); + + // Assert + Assert.Single(missingParams); + Assert.Equal("gender", missingParams[0].Parameter.Name); + Assert.True(foundSurrogateId); + } } } diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlServerSearchService.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlServerSearchService.cs index ec37ba9a2f..dd2321b069 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlServerSearchService.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlServerSearchService.cs @@ -2295,7 +2295,7 @@ private static void PopulateGetResourceSurrogateIdRangesCommand(SqlCommand cmd, cmd.Parameters.AddWithValue("@ActiveOnly", activeOnly); } - private class ResourceSearchParamStats + internal class ResourceSearchParamStats { private readonly ConcurrentDictionary<(string TableName, string ColumnName, short ResourceTypeId, short SearchParamId), bool> _stats; private readonly SearchParamTableExpressionQueryGeneratorFactory _queryGeneratorFactory; @@ -2328,9 +2328,10 @@ public async Task Create( { var tableExpression = expression.SearchParamTableExpressions[tableIndex]; - // We support Normal and Union. Skip include/sort/etc. + // We support Normal, Union, and NotExists. Skip include/sort/etc. if (tableExpression.Kind != SearchParamTableExpressionKind.Normal && - tableExpression.Kind != SearchParamTableExpressionKind.Union) + tableExpression.Kind != SearchParamTableExpressionKind.Union && + tableExpression.Kind != SearchParamTableExpressionKind.NotExists) { continue; } @@ -2338,6 +2339,9 @@ public async Task Create( // Collected raw triples (table, resourceTypeId, searchParamId) var collected = new List<(string Table, short ResourceTypeId, short SearchParamId)>(); + // Track whether we also need a ResourceSurrogateId filtered stat + bool hasResourceSurrogateId = false; + if (tableExpression.Kind == SearchParamTableExpressionKind.Normal) { ProcessPredicateForStats(tableExpression.Predicate, tableExpression.QueryGenerator, model, tableExpression.ChainLevel, expression, tableIndex, collected, logger, parentMultiaryContext: null, isUnionBranch: false); @@ -2351,6 +2355,10 @@ public async Task Create( ProcessUnionBranch(branch, tableExpression.QueryGenerator, model, tableExpression.ChainLevel, expression, tableIndex, collected, logger); } } + else if (tableExpression.Kind == SearchParamTableExpressionKind.NotExists) + { + ProcessNotExistsForStats(tableExpression.Predicate, tableExpression.QueryGenerator, model, collected, out hasResourceSurrogateId, logger); + } // Emit stats rows foreach (var (table, resourceTypeId, searchParamId) in collected) @@ -2365,6 +2373,12 @@ public async Task Create( { await Create(table, column, resourceTypeId, searchParamId, sqlRetryService, logger, cancel); } + + // For NotExists with ResourceSurrogateId range constraint, create an additional filtered stat + if (hasResourceSurrogateId) + { + await Create(table, "ResourceSurrogateId", resourceTypeId, searchParamId, sqlRetryService, logger, cancel); + } } } } @@ -2394,6 +2408,115 @@ private void ProcessUnionBranch( } } + /// + /// Processes a NotExists predicate (produced by MissingSearchParamVisitor for :missing=true queries) + /// to extract the owning search parameter and optionally detect ResourceSurrogateId range constraints. + /// + private void ProcessNotExistsForStats( + Expression predicate, + SearchParamTableExpressionQueryGenerator defaultGenerator, + SqlServerFhirModel model, + List<(string Table, short ResourceTypeId, short SearchParamId)> collected, + out bool hasResourceSurrogateId, + ILogger logger) + { + hasResourceSurrogateId = false; + + var missingParams = new List(); + var resourceTypeIds = new HashSet(); + bool foundSurrogateId = false; + + CollectNotExistsLeaves(predicate, missingParams, resourceTypeIds, model, ref foundSurrogateId); + + // Conservative: skip if predicate resolves to anything other than exactly one owning search parameter + if (missingParams.Count != 1) + { + return; + } + + var missingParam = missingParams[0]; + + // Skip synthetic parameters + if (missingParam.Parameter.Name == SqlSearchParameters.PrimaryKeyParameterName || + missingParam.Parameter.Name == SqlSearchParameters.ResourceSurrogateIdParameterName) + { + return; + } + + // Resolve the table for this parameter + var specificGenerator = missingParam.AcceptVisitor(_queryGeneratorFactory, _queryGeneratorFactory.InitialContext) ?? defaultGenerator; + var tableName = specificGenerator.Table.TableName; + + // Resolve search param ID + if (!model.TryGetSearchParamId(missingParam.Parameter.Url, out var searchParamId) || searchParamId == 0) + { + return; + } + + // Fall back to base resource types if none found in predicate + if (resourceTypeIds.Count == 0 && missingParam.Parameter.BaseResourceTypes?.Count > 0) + { + foreach (var baseType in missingParam.Parameter.BaseResourceTypes) + { + if (model.TryGetResourceTypeId(baseType, out var rtId)) + { + resourceTypeIds.Add(rtId); + } + } + } + + if (resourceTypeIds.Count == 0) + { + return; + } + + hasResourceSurrogateId = foundSurrogateId; + + foreach (var rtId in resourceTypeIds) + { + collected.Add((tableName, rtId, searchParamId)); + } + } + + /// + /// Recursively collects MissingSearchParameterExpression leaves, resource type constraints, + /// and detects ResourceSurrogateId range constraints from a NotExists predicate. + /// + internal static void CollectNotExistsLeaves( + Expression expression, + List missingParams, + HashSet resourceTypeIds, + SqlServerFhirModel model, + ref bool foundSurrogateId) + { + switch (expression) + { + case MissingSearchParameterExpression msp: + missingParams.Add(msp); + break; + + case SearchParameterExpression spe: + if (spe.Parameter.Name == SearchParameterNames.ResourceType) + { + CollectResourceTypesFromExpression(spe.Expression, model, resourceTypeIds); + } + else if (spe.Parameter.Name == SqlSearchParameters.ResourceSurrogateIdParameterName) + { + foundSurrogateId = true; + } + + break; + + case MultiaryExpression multi: + foreach (var inner in multi.Expressions) + { + CollectNotExistsLeaves(inner, missingParams, resourceTypeIds, model, ref foundSurrogateId); + } + + break; + } + } + private void ProcessPredicateForStats( Expression predicate, SearchParamTableExpressionQueryGenerator defaultGenerator,