Support LINQ GroupBy subquery (#2210)

Author: bahusoid

src/NHibernate.Test/Async/Linq/NestedSelectsTests.cs

@@ -362,5 +362,38 @@ public async Task OrdersIdWithOrderLinesNestedWhereIdAsync()
 			Assert.That(orders.Count, Is.EqualTo(830));
 			Assert.That(orders[0].OrderLinesIds, Is.Empty);
 		}
+
+		[Test]
+		public async Task NoNestedSelects_AnyOnGroupBySubqueryAsync()
+		{
+			var subQuery = from vms in db.Animals
+							group vms by vms.Father
+							into vmReqs
+							select vmReqs.Max(x => x.Id);
+
+			var outerQuery = from vm in db.Animals
+							where subQuery.Any(x => vm.Id == x)
+							select vm;
+			var animals = await (outerQuery.ToListAsync());
+			Assert.That(animals.Count, Is.EqualTo(2));
+		}
+
+		//NH-3155
+		[Test]
+		public async Task NoNestedSelects_ContainsOnGroupBySubqueryAsync()
+		{
+			var subQuery = from vms in db.Animals
+							where vms.BodyWeight > 0
+							group vms by vms.Father
+							into vmReqs
+							select vmReqs.Max(x => x.Id);
+
+			var outerQuery = from vm in db.Animals
+							where subQuery.Contains(vm.Id)
+							select vm;
+
+			var animals = await (outerQuery.ToListAsync());
+			Assert.That(animals.Count, Is.EqualTo(2));
+		}
 	}
-}
+}

src/NHibernate.Test/Linq/NestedSelectsTests.cs

@@ -364,5 +364,38 @@ public void OrdersIdWithOrderLinesNestedWhereId()
 			Assert.That(orders.Count, Is.EqualTo(830));
 			Assert.That(orders[0].OrderLinesIds, Is.Empty);
 		}
+
+		[Test]
+		public void NoNestedSelects_AnyOnGroupBySubquery()
+		{
+			var subQuery = from vms in db.Animals
+							group vms by vms.Father
+							into vmReqs
+							select vmReqs.Max(x => x.Id);
+
+			var outerQuery = from vm in db.Animals
+							where subQuery.Any(x => vm.Id == x)
+							select vm;
+			var animals = outerQuery.ToList();
+			Assert.That(animals.Count, Is.EqualTo(2));
+		}
+
+		//NH-3155
+		[Test]
+		public void NoNestedSelects_ContainsOnGroupBySubquery()
+		{
+			var subQuery = from vms in db.Animals
+							where vms.BodyWeight > 0
+							group vms by vms.Father
+							into vmReqs
+							select vmReqs.Max(x => x.Id);
+
+			var outerQuery = from vm in db.Animals
+							where subQuery.Contains(vm.Id)
+							select vm;
+
+			var animals = outerQuery.ToList();
+			Assert.That(animals.Count, Is.EqualTo(2));
+		}
 	}
-}
+}

src/NHibernate/Linq/GroupBy/AggregatingGroupByRewriter.cs

@@ -38,7 +38,8 @@ public static class AggregatingGroupByRewriter
 				typeof(FirstResultOperator),
 				typeof(SingleResultOperator),
 				typeof(AnyResultOperator),
-				typeof(AllResultOperator)
+				typeof(AllResultOperator),
+				typeof(ContainsResultOperator),
 			};
 
 		public static void ReWrite(QueryModel queryModel)

src/NHibernate/Linq/NestedSelects/NestedSelectRewriter.cs

@@ -44,6 +44,9 @@ public static void ReWrite(QueryModel queryModel, ISessionFactory sessionFactory
 					replacements.Add(expression, processed);
 			}
 
+			if (!replacements.Any())
+				return;
+
 			var key = Expression.Property(group, IGroupingKeyProperty);
 
 			var expressions = new List<ExpressionHolder>();
@@ -92,6 +95,10 @@ private static Expression ProcessExpression(QueryModel queryModel, ISessionFacto
 
 		private static Expression ProcessSubquery(ISessionFactory sessionFactory, ICollection<ExpressionHolder> elementExpression, QueryModel queryModel, Expression @group, QueryModel subQueryModel)
 		{
+			var resultTypeOverride = subQueryModel.ResultTypeOverride;
+			if (resultTypeOverride != null && !resultTypeOverride.IsArray && !resultTypeOverride.IsEnumerableOfT())
+				return null;
+
 			var subQueryMainFromClause = subQueryModel.MainFromClause;
 
 			var restrictions = subQueryModel.BodyClauses

src/NHibernate/Linq/Visitors/QueryModelVisitor.cs

@@ -365,7 +365,7 @@ public override void VisitNhJoinClause(NhJoinClause joinClause, QueryModel query
 		public override void VisitResultOperator(ResultOperatorBase resultOperator, QueryModel queryModel, int index)
 		{
 			PreviousEvaluationType = CurrentEvaluationType;
-			CurrentEvaluationType = resultOperator.GetOutputDataInfo(PreviousEvaluationType);
+			CurrentEvaluationType = GetOutputDataInfo(resultOperator, PreviousEvaluationType);
 
 			if (resultOperator is ClientSideTransformOperator)
 			{
@@ -382,6 +382,26 @@ public override void VisitResultOperator(ResultOperatorBase resultOperator, Quer
 			ResultOperatorMap.Process(resultOperator, this, _hqlTree);
 		}
 
+		private static IStreamedDataInfo GetOutputDataInfo(ResultOperatorBase resultOperator, IStreamedDataInfo evaluationType)
+		{
+			//ContainsResultOperator contains data integrity check so for `values.Contains(x)` it checks that  'x' is proper type to be used inside 'values.Contains()'
+			//Due to some reasons (possibly NH expression rewritings) those types might be incompatible (known case NH-3155 - group by subquery). So resultOperator.GetOutputDataInfo throws something like: 
+			//System.ArgumentException : The items of the input sequence of type 'System.Linq.IGrouping`2[System.Object[],EntityType]' are not compatible with the item expression of type 'System.Int32'.
+			//Parameter name: inputInfo
+			//at Remotion.Linq.Clauses.ResultOperators.ContainsResultOperator.GetOutputDataInfo(StreamedSequenceInfo inputInfo)
+			//But in this place we don't really care about types involving inside expression, all we need to know is operation result which is bool for Contains
+			//So let's skip possible type exception mismatch if it allows to generate proper SQL
+			switch (resultOperator)
+			{
+				case ContainsResultOperator _:
+				case AnyResultOperator _:
+				case AllResultOperator _:
+					return new StreamedScalarValueInfo(typeof(bool));
+			}
+
+			return resultOperator.GetOutputDataInfo(evaluationType);
+		}
+
 		public override void VisitSelectClause(SelectClause selectClause, QueryModel queryModel)
 		{
 			CurrentEvaluationType = selectClause.GetOutputDataInfo();