diff --git a/src/System.Linq.Dynamic.Core.NewtonsoftJson/Config/NewtonsoftJsonParsingConfig.cs b/src/System.Linq.Dynamic.Core.NewtonsoftJson/Config/NewtonsoftJsonParsingConfig.cs index fad7f9f1..fbccbf64 100644 --- a/src/System.Linq.Dynamic.Core.NewtonsoftJson/Config/NewtonsoftJsonParsingConfig.cs +++ b/src/System.Linq.Dynamic.Core.NewtonsoftJson/Config/NewtonsoftJsonParsingConfig.cs @@ -15,5 +15,20 @@ public class NewtonsoftJsonParsingConfig : ParsingConfig /// /// The default to use. /// - public DynamicJsonClassOptions? DynamicJsonClassOptions { get; set; } + public DynamicJsonClassOptions? DynamicJsonClassOptions { get; set; } + + /// + /// Gets or sets a value indicating whether the objects in an array should be normalized before processing. + /// + public bool Normalize { get; set; } = true; + + /// + /// Gets or sets the behavior to apply when a property value does not exist during normalization. + /// + /// + /// Use this property to control how the normalization process handles properties that are missing or undefined. + /// The selected behavior may affect the output or error handling of normalization operations. + /// The default value is . + /// + public NormalizationNonExistingPropertyBehavior NormalizationNonExistingPropertyValueBehavior { get; set; } } \ No newline at end of file diff --git a/src/System.Linq.Dynamic.Core.NewtonsoftJson/Config/NormalizationNonExistingPropertyBehavior.cs b/src/System.Linq.Dynamic.Core.NewtonsoftJson/Config/NormalizationNonExistingPropertyBehavior.cs new file mode 100644 index 00000000..bb68277b --- /dev/null +++ b/src/System.Linq.Dynamic.Core.NewtonsoftJson/Config/NormalizationNonExistingPropertyBehavior.cs @@ -0,0 +1,17 @@ +namespace System.Linq.Dynamic.Core.NewtonsoftJson.Config; + +/// +/// Specifies the behavior to use when setting a property value that does not exist or is missing during normalization. +/// +public enum NormalizationNonExistingPropertyBehavior +{ + /// + /// Specifies that the default value should be used. + /// + UseDefaultValue = 0, + + /// + /// Specifies that null values should be used. + /// + UseNull = 1 +} \ No newline at end of file diff --git a/src/System.Linq.Dynamic.Core.NewtonsoftJson/JsonValueInfo.cs b/src/System.Linq.Dynamic.Core.NewtonsoftJson/JsonValueInfo.cs new file mode 100644 index 00000000..963ff37d --- /dev/null +++ b/src/System.Linq.Dynamic.Core.NewtonsoftJson/JsonValueInfo.cs @@ -0,0 +1,10 @@ +using Newtonsoft.Json.Linq; + +namespace System.Linq.Dynamic.Core.NewtonsoftJson; + +internal readonly struct JsonValueInfo(JTokenType type, object? value) +{ + public JTokenType Type { get; } = type; + + public object? Value { get; } = value; +} \ No newline at end of file diff --git a/src/System.Linq.Dynamic.Core.NewtonsoftJson/NewtonsoftJsonExtensions.cs b/src/System.Linq.Dynamic.Core.NewtonsoftJson/NewtonsoftJsonExtensions.cs index 8aefa397..943df3dd 100644 --- a/src/System.Linq.Dynamic.Core.NewtonsoftJson/NewtonsoftJsonExtensions.cs +++ b/src/System.Linq.Dynamic.Core.NewtonsoftJson/NewtonsoftJsonExtensions.cs @@ -1,6 +1,7 @@ using System.Collections; using System.Linq.Dynamic.Core.NewtonsoftJson.Config; using System.Linq.Dynamic.Core.NewtonsoftJson.Extensions; +using System.Linq.Dynamic.Core.NewtonsoftJson.Utils; using System.Linq.Dynamic.Core.Validation; using JetBrains.Annotations; using Newtonsoft.Json.Linq; @@ -870,7 +871,13 @@ private static JArray ToJArray(Func func) private static IQueryable ToQueryable(JArray source, NewtonsoftJsonParsingConfig? config = null) { - return source.ToDynamicJsonClassArray(config?.DynamicJsonClassOptions).AsQueryable(); + var normalized = config?.Normalize == true ? + NormalizeUtils.NormalizeArray(source, config.NormalizationNonExistingPropertyValueBehavior): + source; + + return normalized + .ToDynamicJsonClassArray(config?.DynamicJsonClassOptions) + .AsQueryable(); } #endregion } \ No newline at end of file diff --git a/src/System.Linq.Dynamic.Core.NewtonsoftJson/Utils/NormalizeUtils.cs b/src/System.Linq.Dynamic.Core.NewtonsoftJson/Utils/NormalizeUtils.cs new file mode 100644 index 00000000..5669915c --- /dev/null +++ b/src/System.Linq.Dynamic.Core.NewtonsoftJson/Utils/NormalizeUtils.cs @@ -0,0 +1,131 @@ +using System.Collections.Generic; +using System.Linq.Dynamic.Core.NewtonsoftJson.Config; +using Newtonsoft.Json.Linq; + +namespace System.Linq.Dynamic.Core.NewtonsoftJson.Utils; + +internal static class NormalizeUtils +{ + /// + /// Normalizes an array of JSON objects so that each object contains all properties found in the array, + /// including nested objects. Missing properties will have null values. + /// + internal static JArray NormalizeArray(JArray jsonArray, NormalizationNonExistingPropertyBehavior normalizationBehavior) + { + if (jsonArray.Any(item => item is not JObject)) + { + return jsonArray; + } + + var schema = BuildSchema(jsonArray); + var normalizedArray = new JArray(); + + foreach (var jo in jsonArray.OfType()) + { + var normalizedObj = NormalizeObject(jo, schema, normalizationBehavior); + normalizedArray.Add(normalizedObj); + } + + return normalizedArray; + } + + private static Dictionary BuildSchema(JArray array) + { + var schema = new Dictionary(); + + foreach (var item in array) + { + if (item is JObject obj) + { + MergeSchema(schema, obj); + } + } + + return schema; + } + + private static void MergeSchema(Dictionary schema, JObject obj) + { + foreach (var prop in obj.Properties()) + { + if (prop.Value is JObject nested) + { + if (!schema.TryGetValue(prop.Name, out var jsonValueInfo)) + { + jsonValueInfo = new JsonValueInfo(JTokenType.Object, new Dictionary()); + schema[prop.Name] = jsonValueInfo; + } + + MergeSchema((Dictionary)jsonValueInfo.Value!, nested); + } + else + { + if (!schema.ContainsKey(prop.Name)) + { + schema[prop.Name] = new JsonValueInfo(prop.Value.Type, null); + } + } + } + } + + private static JObject NormalizeObject(JObject source, Dictionary schema, NormalizationNonExistingPropertyBehavior normalizationBehavior) + { + var result = new JObject(); + + foreach (var key in schema.Keys) + { + if (schema[key].Value is Dictionary nestedSchema) + { + result[key] = source.ContainsKey(key) && source[key] is JObject jo ? NormalizeObject(jo, nestedSchema, normalizationBehavior) : CreateEmptyObject(nestedSchema, normalizationBehavior); + } + else + { + if (source.ContainsKey(key)) + { + result[key] = source[key]; + } + else + { + result[key] = normalizationBehavior == NormalizationNonExistingPropertyBehavior.UseDefaultValue ? GetDefaultValue(schema[key]) : JValue.CreateNull(); + } + } + } + + return result; + } + + private static JObject CreateEmptyObject(Dictionary schema, NormalizationNonExistingPropertyBehavior normalizationBehavior) + { + var obj = new JObject(); + foreach (var key in schema.Keys) + { + if (schema[key].Value is Dictionary nestedSchema) + { + obj[key] = CreateEmptyObject(nestedSchema, normalizationBehavior); + } + else + { + obj[key] = normalizationBehavior == NormalizationNonExistingPropertyBehavior.UseDefaultValue ? GetDefaultValue(schema[key]) : JValue.CreateNull(); + } + } + + return obj; + } + + private static JToken GetDefaultValue(JsonValueInfo jType) + { + return jType.Type switch + { + JTokenType.Array => new JArray(), + JTokenType.Boolean => default(bool), + JTokenType.Bytes => new byte[0], + JTokenType.Date => DateTime.MinValue, + JTokenType.Float => default(float), + JTokenType.Guid => Guid.Empty, + JTokenType.Integer => default(int), + JTokenType.String => string.Empty, + JTokenType.TimeSpan => TimeSpan.MinValue, + _ => JValue.CreateNull(), + }; + } +} \ No newline at end of file diff --git a/src/System.Linq.Dynamic.Core.SystemTextJson/Config/NormalizationNonExistingPropertyBehavior.cs b/src/System.Linq.Dynamic.Core.SystemTextJson/Config/NormalizationNonExistingPropertyBehavior.cs new file mode 100644 index 00000000..daafaa4b --- /dev/null +++ b/src/System.Linq.Dynamic.Core.SystemTextJson/Config/NormalizationNonExistingPropertyBehavior.cs @@ -0,0 +1,17 @@ +namespace System.Linq.Dynamic.Core.SystemTextJson.Config; + +/// +/// Specifies the behavior to use when setting a property vlue that does not exist or is missing during normalization. +/// +public enum NormalizationNonExistingPropertyBehavior +{ + /// + /// Specifies that the default value should be used. + /// + UseDefaultValue = 0, + + /// + /// Specifies that null values should be used. + /// + UseNull = 1 +} \ No newline at end of file diff --git a/src/System.Linq.Dynamic.Core.SystemTextJson/Config/SystemTextJsonParsingConfig.cs b/src/System.Linq.Dynamic.Core.SystemTextJson/Config/SystemTextJsonParsingConfig.cs index eb42ff6d..a4c1e76a 100644 --- a/src/System.Linq.Dynamic.Core.SystemTextJson/Config/SystemTextJsonParsingConfig.cs +++ b/src/System.Linq.Dynamic.Core.SystemTextJson/Config/SystemTextJsonParsingConfig.cs @@ -8,5 +8,23 @@ public class SystemTextJsonParsingConfig : ParsingConfig /// /// The default ParsingConfig for . /// - public new static SystemTextJsonParsingConfig Default { get; } = new(); + public new static SystemTextJsonParsingConfig Default { get; } = new SystemTextJsonParsingConfig + { + ConvertObjectToSupportComparison = true + }; + + /// + /// Gets or sets a value indicating whether the objects in an array should be normalized before processing. + /// + public bool Normalize { get; set; } = true; + + /// + /// Gets or sets the behavior to apply when a property value does not exist during normalization. + /// + /// + /// Use this property to control how the normalization process handles properties that are missing or undefined. + /// The selected behavior may affect the output or error handling of normalization operations. + /// The default value is . + /// + public NormalizationNonExistingPropertyBehavior NormalizationNonExistingPropertyValueBehavior { get; set; } } \ No newline at end of file diff --git a/src/System.Linq.Dynamic.Core.SystemTextJson/Extensions/JsonValueExtensions.cs b/src/System.Linq.Dynamic.Core.SystemTextJson/Extensions/JsonValueExtensions.cs new file mode 100644 index 00000000..907f7bf3 --- /dev/null +++ b/src/System.Linq.Dynamic.Core.SystemTextJson/Extensions/JsonValueExtensions.cs @@ -0,0 +1,21 @@ +#if !NET8_0_OR_GREATER +namespace System.Text.Json.Nodes; + +internal static class JsonValueExtensions +{ + internal static JsonValueKind? GetValueKind(this JsonNode node) + { + if (node is JsonObject) + { + return JsonValueKind.Object; + } + + if (node is JsonArray) + { + return JsonValueKind.Array; + } + + return node.GetValue() is JsonElement je ? je.ValueKind : null; + } +} +#endif \ No newline at end of file diff --git a/src/System.Linq.Dynamic.Core.SystemTextJson/JsonValueInfo.cs b/src/System.Linq.Dynamic.Core.SystemTextJson/JsonValueInfo.cs new file mode 100644 index 00000000..da4f2b5d --- /dev/null +++ b/src/System.Linq.Dynamic.Core.SystemTextJson/JsonValueInfo.cs @@ -0,0 +1,10 @@ +using System.Text.Json; + +namespace System.Linq.Dynamic.Core.SystemTextJson; + +internal readonly struct JsonValueInfo(JsonValueKind type, object? value) +{ + public JsonValueKind Type { get; } = type; + + public object? Value { get; } = value; +} \ No newline at end of file diff --git a/src/System.Linq.Dynamic.Core.SystemTextJson/SystemTextJsonExtensions.cs b/src/System.Linq.Dynamic.Core.SystemTextJson/SystemTextJsonExtensions.cs index 8d1671a0..6d8a5aa0 100644 --- a/src/System.Linq.Dynamic.Core.SystemTextJson/SystemTextJsonExtensions.cs +++ b/src/System.Linq.Dynamic.Core.SystemTextJson/SystemTextJsonExtensions.cs @@ -1093,13 +1093,20 @@ private static JsonDocument ToJsonDocumentArray(Func func) // ReSharper disable once UnusedParameter.Local private static IQueryable ToQueryable(JsonDocument source, SystemTextJsonParsingConfig? config = null) { + config = config ?? SystemTextJsonParsingConfig.Default; + config.ConvertObjectToSupportComparison = true; + var array = source.RootElement; if (array.ValueKind != JsonValueKind.Array) { throw new NotSupportedException("The source is not a JSON array."); } - return JsonDocumentExtensions.ToDynamicJsonClassArray(array).AsQueryable(); + var normalized = config.Normalize ? + NormalizeUtils.NormalizeJsonDocument(source, config.NormalizationNonExistingPropertyValueBehavior) : + source; + + return JsonDocumentExtensions.ToDynamicJsonClassArray(normalized.RootElement).AsQueryable(); } #endregion } \ No newline at end of file diff --git a/src/System.Linq.Dynamic.Core.SystemTextJson/Utils/NormalizeUtils.cs b/src/System.Linq.Dynamic.Core.SystemTextJson/Utils/NormalizeUtils.cs new file mode 100644 index 00000000..fa5836d2 --- /dev/null +++ b/src/System.Linq.Dynamic.Core.SystemTextJson/Utils/NormalizeUtils.cs @@ -0,0 +1,156 @@ +using System.Collections.Generic; +using System.Linq.Dynamic.Core.SystemTextJson.Config; +using System.Text.Json; +using System.Text.Json.Nodes; + +namespace System.Linq.Dynamic.Core.SystemTextJson.Utils; + +internal static class NormalizeUtils +{ + /// + /// Normalizes a document so that each object contains all properties found in the array, including nested objects. + /// + internal static JsonDocument NormalizeJsonDocument(JsonDocument jsonDocument, NormalizationNonExistingPropertyBehavior normalizationBehavior) + { + if (jsonDocument.RootElement.ValueKind != JsonValueKind.Array) + { + throw new NotSupportedException("The source is not a JSON array."); + } + + var jsonArray = JsonNode.Parse(jsonDocument.RootElement.GetRawText())!.AsArray(); + var normalizedArray = NormalizeJsonArray(jsonArray, normalizationBehavior); + + return JsonDocument.Parse(normalizedArray.ToJsonString()); + } + + /// + /// Normalizes an array of JSON objects so that each object contains all properties found in the array, including nested objects. + /// + internal static JsonArray NormalizeJsonArray(JsonArray jsonArray, NormalizationNonExistingPropertyBehavior normalizationBehavior) + { + if (jsonArray.Any(item => item != null && item.GetValueKind() != JsonValueKind.Object)) + { + return jsonArray; + } + + var schema = BuildSchema(jsonArray); + var normalizedArray = new JsonArray(); + + foreach (var item in jsonArray) + { + if (item is JsonObject obj) + { + var normalizedObj = NormalizeObject(obj, schema, normalizationBehavior); + normalizedArray.Add(normalizedObj); + } + } + + return normalizedArray; + } + + private static Dictionary BuildSchema(JsonArray array) + { + var schema = new Dictionary(); + + foreach (var item in array) + { + if (item is JsonObject obj) + { + MergeSchema(schema, obj); + } + } + + return schema; + } + + private static void MergeSchema(Dictionary schema, JsonObject obj) + { + foreach (var prop in obj) + { + if (prop.Value is JsonObject nested) + { + if (!schema.TryGetValue(prop.Key, out var jsonValueInfo)) + { + jsonValueInfo = new JsonValueInfo(JsonValueKind.Object, new Dictionary()); + schema[prop.Key] = jsonValueInfo; + } + + MergeSchema((Dictionary)jsonValueInfo.Value!, nested); + } + else + { + if (!schema.ContainsKey(prop.Key)) + { + schema[prop.Key] = new JsonValueInfo(prop.Value?.GetValueKind() ?? JsonValueKind.Null, null); + } + } + } + } + + private static JsonObject NormalizeObject(JsonObject source, Dictionary schema, NormalizationNonExistingPropertyBehavior normalizationBehavior) + { + var result = new JsonObject(); + + foreach (var kvp in schema) + { + var key = kvp.Key; + var jType = kvp.Value; + + if (jType.Value is Dictionary nestedSchema) + { + result[key] = source.ContainsKey(key) && source[key] is JsonObject jo ? NormalizeObject(jo, nestedSchema, normalizationBehavior) : CreateEmptyObject(nestedSchema, normalizationBehavior); + } + else + { + if (source.ContainsKey(key)) + { +#if NET8_0_OR_GREATER + result[key] = source[key]!.DeepClone(); +#else + result[key] = JsonNode.Parse(source[key]!.ToJsonString()); +#endif + } + else + { + result[key] = normalizationBehavior == NormalizationNonExistingPropertyBehavior.UseDefaultValue ? GetDefaultValue(jType) : null; + } + } + } + + return result; + } + + private static JsonObject CreateEmptyObject(Dictionary schema, NormalizationNonExistingPropertyBehavior normalizationBehavior) + { + var obj = new JsonObject(); + foreach (var kvp in schema) + { + var key = kvp.Key; + var jType = kvp.Value; + + if (jType.Value is Dictionary nestedSchema) + { + obj[key] = CreateEmptyObject(nestedSchema, normalizationBehavior); + } + else + { + obj[key] = normalizationBehavior == NormalizationNonExistingPropertyBehavior.UseDefaultValue ? GetDefaultValue(jType) : null; + } + } + + return obj; + } + + private static JsonNode? GetDefaultValue(JsonValueInfo jType) + { + return jType.Type switch + { + JsonValueKind.Array => new JsonArray(), + JsonValueKind.False => false, + JsonValueKind.Number => default(int), + JsonValueKind.String => string.Empty, + JsonValueKind.True => false, + _ => null, + }; + } +} \ No newline at end of file diff --git a/test/System.Linq.Dynamic.Core.NewtonsoftJson.Tests/NewtonsoftJsonTests.cs b/test/System.Linq.Dynamic.Core.NewtonsoftJson.Tests/NewtonsoftJsonTests.cs index 40185d45..e391ed29 100644 --- a/test/System.Linq.Dynamic.Core.NewtonsoftJson.Tests/NewtonsoftJsonTests.cs +++ b/test/System.Linq.Dynamic.Core.NewtonsoftJson.Tests/NewtonsoftJsonTests.cs @@ -1,4 +1,5 @@ -using System.Linq.Dynamic.Core.NewtonsoftJson.Config; +using System.Linq.Dynamic.Core.Exceptions; +using System.Linq.Dynamic.Core.NewtonsoftJson.Config; using FluentAssertions; using Newtonsoft.Json.Linq; using Xunit; @@ -508,36 +509,6 @@ public void Where_With_Select() first.Value().Should().Be("Doe"); } - //[Fact] - //public void Where_OptionalProperty() - //{ - // // Arrange - // var config = new NewtonsoftJsonParsingConfig - // { - // ConvertObjectToSupportComparison = true - // }; - // var array = - // """ - // [ - // { - // "Name": "John", - // "Age": 30 - // }, - // { - // "Name": "Doe" - // } - // ] - // """; - - // // Act - // var result = JArray.Parse(array).Where(config, "Age > 30").Select("Name"); - - // // Assert - // result.Should().HaveCount(1); - // var first = result.First(); - // first.Value().Should().Be("John"); - //} - [Theory] [InlineData("notExisting == true")] [InlineData("notExisting == \"true\"")] @@ -565,4 +536,37 @@ public void Where_NonExistingMember_EmptyResult(string predicate) // Assert result.Should().BeEmpty(); } + + [Theory] + [InlineData("""[ { "Name": "John", "Age": 30 }, { "Name": "Doe" }, { } ]""")] + [InlineData("""[ { "Name": "Doe" }, { "Name": "John", "Age": 30 }, { } ]""")] + public void NormalizeArray(string array) + { + // Act + var result = JArray.Parse(array) + .Where("Age >= 30") + .Select("Name"); + + // Assert + result.Should().HaveCount(1); + var first = result.First(); + first.Value().Should().Be("John"); + } + + [Fact] + public void NormalizeArray_When_NormalizeIsFalse_ShouldThrow() + { + // Arrange + var config = new NewtonsoftJsonParsingConfig + { + Normalize = false + }; + var array = """[ { "Name": "Doe" }, { "Name": "John", "Age": 30 }, { } ]"""; + + // Act + Action act = () => JArray.Parse(array).Where(config, "Age >= 30"); + + // Assert + act.Should().Throw().WithMessage("The binary operator GreaterThanOrEqual is not defined for the types 'System.Object' and 'System.Int32'."); + } } \ No newline at end of file diff --git a/test/System.Linq.Dynamic.Core.SystemTextJson.Tests/SystemTextJsonTests.cs b/test/System.Linq.Dynamic.Core.SystemTextJson.Tests/SystemTextJsonTests.cs index 1e817664..7b27ec5c 100644 --- a/test/System.Linq.Dynamic.Core.SystemTextJson.Tests/SystemTextJsonTests.cs +++ b/test/System.Linq.Dynamic.Core.SystemTextJson.Tests/SystemTextJsonTests.cs @@ -551,17 +551,48 @@ public void Where_With_Select() [InlineData("\"something\" == notExisting")] [InlineData("1 < notExisting")] public void Where_NonExistingMember_EmptyResult(string predicate) + { + // Act + var result = _source.Where(predicate).RootElement.EnumerateArray(); + + // Assert + result.Should().BeEmpty(); + } + + [Theory] + [InlineData("""[ { "Name": "John", "Age": 30 }, { "Name": "Doe" }, { } ]""")] + [InlineData("""[ { "Name": "Doe" }, { "Name": "John", "Age": 30 }, { } ]""")] + public void NormalizeArray(string data) + { + // Arrange + var source = JsonDocument.Parse(data); + + // Act + var result = source + .Where("Age >= 30") + .Select("Name"); + + // Assert + var array = result.RootElement.EnumerateArray(); + array.Should().HaveCount(1); + var first = result.First(); + array.First().GetString().Should().Be("John"); + } + + [Fact] + public void NormalizeArray_When_NormalizeIsFalse_ShouldThrow() { // Arrange var config = new SystemTextJsonParsingConfig { - ConvertObjectToSupportComparison = true + Normalize = false }; + var data = """[ { "Name": "Doe" }, { "Name": "John", "Age": 30 }, { } ]"""; // Act - var result = _source.Where(config, predicate).RootElement.EnumerateArray(); + Action act = () => JsonDocument.Parse(data).Where(config, "Age >= 30"); // Assert - result.Should().BeEmpty(); + act.Should().Throw().WithMessage("Unable to find property 'Age' on type '<>f__AnonymousType*"); } } \ No newline at end of file