Changed dynamic linq filter implementation to pass a datetime object instead of a string

Author: Niall Langley

DataPipelineTools.Tests/Common/FilterTests/GetDynamicLinqValueTests.cs

@@ -1,4 +1,5 @@
-using NUnit.Framework;
+using System;
+using NUnit.Framework;
 using SqlCollaborative.Azure.DataPipelineTools.Common;
 
 namespace DataPipelineTools.Tests.Common.FilterTests
@@ -18,7 +19,9 @@ public void Given_Value_And_OpertatorIsNotLike_Should_Return_Value()
             var filter = new Filter<TestPoco>
             {
                 Operator = "eq",
-                Value = "Someone"
+                Value = "Someone",
+                PropertyType = typeof(string),
+                IsValid = true
             };
 
             var result = filter.GetDynamicLinqValue();
@@ -33,7 +36,9 @@ public void Given_ValueWithoutWildcards_And_OpertatorIsLike_Should_Return_Value(
             var filter = new Filter<TestPoco>
             {
                 Operator = "like",
-                Value = "Someone"
+                Value = "Someone",
+                PropertyType = typeof(string),
+                IsValid = true
             };
 
             var result = filter.GetDynamicLinqValue();
@@ -48,13 +53,72 @@ public void Given_ValueWithWildcards_And_OpertatorIsLike_Should_Return_ValueWith
             var filter = new Filter<TestPoco>
             {
                 Operator = "like",
-                Value = "Some*ne"
+                Value = "Some*ne",
+                PropertyType = typeof(string),
+                IsValid = true
             };
 
             var result = filter.GetDynamicLinqValue();
 
 
             Assert.That(result, Is.EqualTo("Some.*ne"));
         }
+
+        [Test]
+        public void Given_DateTimeValue_Should_Return_ValueOfTypeDateTime()
+        {
+            var date = "2021-01-01T00:00:00";
+            var filter = new Filter<TestPoco>
+            {
+                Operator = "like",
+                Value = date,
+                PropertyType = typeof(DateTime),
+                IsValid = true
+            };
+
+            var result = filter.GetDynamicLinqValue();
+
+
+            Assert.That(result, Is.EqualTo(DateTime.Parse(date)));
+        }
+
+
+
+        [Test]
+        public void Given_DateTimeOffsetValue_Should_Return_ValueOfTypeDateTimeOffset()
+        {
+            var date = "2021-01-01T00:00:00+06:00";
+            var filter = new Filter<TestPoco>
+            {
+                Operator = "like",
+                Value = date,
+                PropertyType = typeof(DateTimeOffset),
+                IsValid = true
+            };
+
+            var result = filter.GetDynamicLinqValue();
+
+
+            Assert.That(result, Is.EqualTo(DateTimeOffset.Parse(date)));
+        }
+
+
+
+        [Test]
+        public void Given_InvalidFilter_Should_Return_Null()
+        {
+            var filter = new Filter<TestPoco>
+            {
+                Operator = "like",
+                Value = "2021-01-01T00:00:00",
+                PropertyType = typeof(DateTime),
+                IsValid = false
+            };
+
+            var result = filter.GetDynamicLinqValue();
+
+            Assert.That(result, Is.Null);
+        }
+
     }
 }

DataPipelineTools/Common/Filter.cs

@@ -12,6 +12,7 @@ namespace SqlCollaborative.Azure.DataPipelineTools.Common
     public class Filter<T>
     {
         public string PropertyName { get; set; }
+        public Type PropertyType { get; set; }
         public string Operator { get; set; }
         public string Value { get; set; }
         public bool IsValid { get; set; } = false;
@@ -29,14 +30,27 @@ public string GetDynamicLinqString()
             }
         }
 
-        public string GetDynamicLinqValue()
+        public object GetDynamicLinqValue()
         {
-            return Operator == "like" ? Value.Replace("*", ".*") : Value;
+            // Avoid throwing trying to do an invalid cast
+            if (!IsValid)
+                return null;
+
+            switch (PropertyType.Name)
+            {
+                case nameof(DateTime):
+                    return DateTime.Parse(Value);
+                case nameof(DateTimeOffset):
+                    return DateTimeOffset.Parse(Value);
+                default:
+                    return Operator == "like" ? Value.Replace("*", ".*") : Value;
+            }
         }
 
 
         #region Newtonsoft.Json serialization methods
         public bool ShouldSerializeIsValid() => false;
+        public bool ShouldSerializePropertyType() => false;
         #endregion Newtonsoft.Json serialization methods
     }
 }

DataPipelineTools/Common/FilterFactory.cs

@@ -54,8 +54,8 @@ public static Filter<T> Create(string columnName, string filter, ILogger log)
             var val = operatorMatches.FirstOrDefault().Groups[2].Value;
 
             // If the operator is like, but it's not a string type it's not a valid filter
-            var propertyType = targetProperty.PropertyType.Name;
-            if (propertyType != nameof(String) && op == "like")
+            var propertyType = targetProperty.PropertyType;
+            if (propertyType.Name != nameof(String) && op == "like")
             {
                 var error = $"The filter column '{columnName}' is not a string, and cannot be use with the 'like' operator.";
                 log?.LogWarning(error);
@@ -68,7 +68,7 @@ public static Filter<T> Create(string columnName, string filter, ILogger log)
 
             // Now we check the filter string can be parsed into the correct type
             var isValueParseable = false;
-            switch (propertyType)
+            switch (propertyType.Name)
             {
                 case nameof(String):
                     isValueParseable = true;
@@ -98,13 +98,14 @@ public static Filter<T> Create(string columnName, string filter, ILogger log)
                     break;
             }
 
-            var parseError = isValueParseable ? null : $"The filter '{val}' cannot be applied to the property '{columnName}' as it cannot be cast to a '{propertyType}'";
+            var parseError = isValueParseable ? null : $"The filter '{val}' cannot be applied to the property '{columnName}' as it cannot be cast to a '{propertyType.Name}'";
             if (!isValueParseable)
                 log?.LogWarning(parseError);
 
             return new Filter<T>
             {
                 PropertyName = columnName,
+                PropertyType = propertyType,
                 Operator = op,
                 Value = val,
                 IsValid = isValueParseable,

DataPipelineTools/DataLake/DataLakeService.cs

@@ -129,7 +129,7 @@ await CheckPathAsync(getItemsConfig.Directory, true) :
             foreach (var filter in getItemsConfig.Filters.Where(f => f.IsValid))
             {
                 var dynamicLinqQuery = filter.GetDynamicLinqString();
-                string dynamicLinqQueryValue = filter.GetDynamicLinqValue();
+                var dynamicLinqQueryValue = filter.GetDynamicLinqValue();
                 _logger.LogInformation($"Applying filter: paths.AsQueryable().Where(\"{dynamicLinqQuery}\", \"{filter.Value}\").ToList()");
                 paths = paths.AsQueryable().Where(dynamicLinqQuery, dynamicLinqQueryValue).ToList();
             }