Json : Normalize

Author: StefH

src/System.Linq.Dynamic.Core.NewtonsoftJson/Config/NewtonsoftJsonParsingConfig.cs

@@ -15,5 +15,20 @@ public class NewtonsoftJsonParsingConfig : ParsingConfig
     /// <summary>
     /// The default <see cref="DynamicJsonClassOptions"/> to use.
     /// </summary>
-    public DynamicJsonClassOptions? DynamicJsonClassOptions { get; set; } 
+    public DynamicJsonClassOptions? DynamicJsonClassOptions { get; set; }
+
+    /// <summary>
+    /// Gets or sets a value indicating whether the objecs in an array should be normalized before processing.
+    /// </summary>
+    public bool Normalize { get; set; } = true;
+
+    /// <summary>
+    /// Gets or sets the behavior to apply when a property value does not exist during normalization.
+    /// </summary>
+    /// <remarks>
+    /// Use this property to control how the normalization process handles properties that are missing or undefined.
+    /// The selected behavior may affect the output or error handling of normalization operations.
+    /// The default value is <see cref="NormalizationNonExistingPropertyValueBehavior.UseDefaultValue"/>.
+    /// </remarks>
+    public NormalizationNonExistingPropertyValueBehavior NormalizationNonExistingPropertyValueBehavior { get; set; }
 }

src/System.Linq.Dynamic.Core.NewtonsoftJson/Config/NormalizationNonExistingPropertyValueBehavior.cs

@@ -0,0 +1,17 @@
+namespace System.Linq.Dynamic.Core.NewtonsoftJson.Config;
+
+/// <summary>
+/// Specifies the behavior to use when setting a property vlue that does not exist or is missing during normalization.
+/// </summary>
+public enum NormalizationNonExistingPropertyValueBehavior
+{
+    /// <summary>
+    /// Specifies that the default value should be used.
+    /// </summary>
+    UseDefaultValue = 0,
+
+    /// <summary>
+    /// Specifies that null values should be used.
+    /// </summary>
+    UseNull = 1
+}

src/System.Linq.Dynamic.Core.NewtonsoftJson/JsonValueInfo.cs

@@ -0,0 +1,10 @@
+using Newtonsoft.Json.Linq;
+
+namespace ConsoleApp3;
+
+internal struct JsonValueInfo(JTokenType type,  object? value)
+{
+    public JTokenType Type { get; } = type;
+
+    public object? Value { get; } = value;
+}

src/System.Linq.Dynamic.Core.NewtonsoftJson/NewtonsoftJsonExtensions.cs

@@ -1,6 +1,7 @@
 using System.Collections;
 using System.Linq.Dynamic.Core.NewtonsoftJson.Config;
 using System.Linq.Dynamic.Core.NewtonsoftJson.Extensions;
+using System.Linq.Dynamic.Core.NewtonsoftJson.Utils;
 using System.Linq.Dynamic.Core.Validation;
 using JetBrains.Annotations;
 using Newtonsoft.Json.Linq;
@@ -870,7 +871,13 @@ private static JArray ToJArray(Func<IQueryable> func)
 
     private static IQueryable ToQueryable(JArray source, NewtonsoftJsonParsingConfig? config = null)
     {
-        return source.ToDynamicJsonClassArray(config?.DynamicJsonClassOptions).AsQueryable();
+        var normalized = config?.Normalize == true ?
+            NormalizeUtils.NormalizeArray(source, config.NormalizationNonExistingPropertyValueBehavior):
+            source;
+
+        return normalized
+            .ToDynamicJsonClassArray(config?.DynamicJsonClassOptions)
+            .AsQueryable();
     }
     #endregion
 }

src/System.Linq.Dynamic.Core.NewtonsoftJson/Utils/NormalizeUtils.cs

@@ -0,0 +1,131 @@
+using System.Collections.Generic;
+using System.Linq.Dynamic.Core.NewtonsoftJson.Config;
+using ConsoleApp3;
+using Newtonsoft.Json.Linq;
+
+namespace System.Linq.Dynamic.Core.NewtonsoftJson.Utils;
+
+internal static class NormalizeUtils
+{
+    /// <summary>
+    /// Normalizes an array of JSON objects so that each object contains all properties found in the array,
+    /// including nested objects. Missing properties will have null values.
+    /// </summary>
+    internal static JArray NormalizeArray(JArray jsonArray, NormalizationNonExistingPropertyValueBehavior normalizationBehavior)
+    {
+        if (jsonArray.Count(item => item is not JObject) > 0)
+        {
+            return jsonArray;
+        }
+
+        var schema = BuildSchema(jsonArray);
+        var normalizedArray = new JArray();
+
+        foreach (var jo in jsonArray.OfType<JObject>())
+        {
+            var normalizedObj = NormalizeObject(jo, schema, normalizationBehavior);
+            normalizedArray.Add(normalizedObj);
+        }
+
+        return normalizedArray;
+    }
+
+    private static Dictionary<string, JsonValueInfo> BuildSchema(JArray array)
+    {
+        var schema = new Dictionary<string, JsonValueInfo>();
+
+        foreach (var item in array)
+        {
+            if (item is JObject obj)
+            {
+                MergeSchema(schema, obj);
+            }
+        }
+
+        return schema;
+    }
+
+    private static void MergeSchema(Dictionary<string, JsonValueInfo> schema, JObject obj)
+    {
+        foreach (var prop in obj.Properties())
+        {
+            if (prop.Value is JObject nested)
+            {
+                if (!schema.ContainsKey(prop.Name))
+                {
+                    schema[prop.Name] = new JsonValueInfo(JTokenType.Object, new Dictionary<string, JsonValueInfo>());
+                }
+
+                MergeSchema((Dictionary<string, JsonValueInfo>)schema[prop.Name].Value!, nested);
+            }
+            else
+            {
+                if (!schema.ContainsKey(prop.Name))
+                {
+                    schema[prop.Name] = new JsonValueInfo(prop.Value.Type, null);
+                }
+            }
+        }
+    }
+
+    private static JObject NormalizeObject(JObject source, Dictionary<string, JsonValueInfo> schema, NormalizationNonExistingPropertyValueBehavior normalizationBehavior)
+    {
+        var result = new JObject();
+
+        foreach (var key in schema.Keys)
+        {
+            if (schema[key].Value is Dictionary<string, JsonValueInfo> nestedSchema)
+            {
+                result[key] = source.ContainsKey(key) && source[key] is JObject jo ? NormalizeObject(jo, nestedSchema, normalizationBehavior) : CreateEmptyObject(nestedSchema, normalizationBehavior);
+            }
+            else
+            {
+                if (source.ContainsKey(key))
+                {
+                    result[key] = source[key];
+                }
+                else
+                {
+                    result[key] = normalizationBehavior == NormalizationNonExistingPropertyValueBehavior.UseDefaultValue ? GetDefaultValue(schema[key]) : JValue.CreateNull();
+                }
+            }
+        }
+
+        return result;
+    }
+
+    private static JObject CreateEmptyObject(Dictionary<string, JsonValueInfo> schema, NormalizationNonExistingPropertyValueBehavior normalizationBehavior)
+    {
+        var obj = new JObject();
+        foreach (var key in schema.Keys)
+        {
+            if (schema[key].Value is Dictionary<string, JsonValueInfo> nestedSchema)
+            {
+                obj[key] = CreateEmptyObject(nestedSchema, normalizationBehavior);
+            }
+            else
+            {
+                obj[key] = normalizationBehavior == NormalizationNonExistingPropertyValueBehavior.UseDefaultValue ? GetDefaultValue(schema[key]) : JValue.CreateNull();
+            }
+        }
+
+        return obj;
+    }
+
+    private static JToken GetDefaultValue(JsonValueInfo jType)
+    {
+        return jType.Type switch
+        {
+            JTokenType.Array => new JArray(),
+            JTokenType.Boolean => default(bool),
+            JTokenType.Bytes => new byte[0],
+            JTokenType.Date => DateTime.MinValue,
+            JTokenType.Float => default(float),
+            JTokenType.Guid => Guid.Empty,
+            JTokenType.Integer => default(int),
+            JTokenType.String => string.Empty,
+            JTokenType.TimeSpan => TimeSpan.MinValue,
+            _ => JValue.CreateNull(),
+        };
+    }
+}

src/System.Linq.Dynamic.Core.SystemTextJson/Config/NormalizationNonExistingPropertyValueBehavior.cs

@@ -0,0 +1,17 @@
+namespace System.Linq.Dynamic.Core.SystemTextJson.Config;
+
+/// <summary>
+/// Specifies the behavior to use when setting a property vlue that does not exist or is missing during normalization.
+/// </summary>
+public enum NormalizationNonExistingPropertyValueBehavior
+{
+    /// <summary>
+    /// Specifies that the default value should be used.
+    /// </summary>
+    UseDefaultValue = 0,
+
+    /// <summary>
+    /// Specifies that null values should be used.
+    /// </summary>
+    UseNull = 1
+}

src/System.Linq.Dynamic.Core.SystemTextJson/Config/SystemTextJsonParsingConfig.cs

@@ -8,5 +8,23 @@ public class SystemTextJsonParsingConfig : ParsingConfig
     /// <summary>
     /// The default ParsingConfig for <see cref="SystemTextJsonParsingConfig"/>.
     /// </summary>
-    public new static SystemTextJsonParsingConfig Default { get; } = new();
+    public new static SystemTextJsonParsingConfig Default { get; } = new SystemTextJsonParsingConfig
+    {
+        ConvertObjectToSupportComparison = true
+    };
+
+    /// <summary>
+    /// Gets or sets a value indicating whether the objecs in an array should be normalized before processing.
+    /// </summary>
+    public bool Normalize { get; set; } = true;
+
+    /// <summary>
+    /// Gets or sets the behavior to apply when a property value does not exist during normalization.
+    /// </summary>
+    /// <remarks>
+    /// Use this property to control how the normalization process handles properties that are missing or undefined.
+    /// The selected behavior may affect the output or error handling of normalization operations.
+    /// The default value is <see cref="NormalizationNonExistingPropertyValueBehavior.UseDefaultValue"/>.
+    /// </remarks>
+    public NormalizationNonExistingPropertyValueBehavior NormalizationNonExistingPropertyValueBehavior { get; set; }
 }

src/System.Linq.Dynamic.Core.SystemTextJson/Extensions/JsonValueExtensions.cs

@@ -0,0 +1,21 @@
+#if !NET8_0_OR_GREATER
+namespace System.Text.Json.Nodes;
+
+internal static class JsonValueExtensions
+{
+    internal static JsonValueKind? GetValueKind(this JsonNode node)
+    {
+        if (node is JsonObject)
+        {
+            return JsonValueKind.Object;
+        }
+
+        if (node is JsonArray)
+        {
+            return JsonValueKind.Array;
+        }
+
+        return node.GetValue<object>() is JsonElement je ? je.ValueKind : null;
+    }
+}
+#endif

src/System.Linq.Dynamic.Core.SystemTextJson/JsonValueInfo.cs

@@ -0,0 +1,8 @@
+using System.Text.Json;
+
+internal struct JsonValueInfo(JsonValueKind type, object? value)
+{
+    public JsonValueKind Type { get; } = type;
+
+    public object? Value { get; } = value;
+}

src/System.Linq.Dynamic.Core.SystemTextJson/SystemTextJsonExtensions.cs

@@ -5,6 +5,7 @@
 using System.Linq.Dynamic.Core.SystemTextJson.Utils;
 using System.Linq.Dynamic.Core.Validation;
 using System.Text.Json;
+using ConsoleApp3;
 using JetBrains.Annotations;
 
 namespace System.Linq.Dynamic.Core.SystemTextJson;
@@ -1093,13 +1094,20 @@ private static JsonDocument ToJsonDocumentArray(Func<IQueryable> func)
     // ReSharper disable once UnusedParameter.Local
     private static IQueryable ToQueryable(JsonDocument source, SystemTextJsonParsingConfig? config = null)
     {
+        config = config ?? SystemTextJsonParsingConfig.Default;
+        config.ConvertObjectToSupportComparison = true;
+
         var array = source.RootElement;
         if (array.ValueKind != JsonValueKind.Array)
         {
             throw new NotSupportedException("The source is not a JSON array.");
         }
 
-        return JsonDocumentExtensions.ToDynamicJsonClassArray(array).AsQueryable();
+        var normalized = config.Normalize ?
+           NormalizeUtils.NormalizeJsonDocument(source, config.NormalizationNonExistingPropertyValueBehavior) :
+           source;
+
+        return JsonDocumentExtensions.ToDynamicJsonClassArray(normalized.RootElement).AsQueryable();
     }
     #endregion
 }