Fix the Serialization/Deserialization issue with $ prefix columns (#2944)

### Why make this change?
Serialization and deserialization of metadata currently fail when column
names are prefixed with the $ symbol.

### Root cause
This issue occurs because we’ve enabled the ReferenceHandler flag in our
System.Text.Json serialization settings. When this flag is active, the
serializer treats $ as a reserved character used for special metadata
(e.g., $id, $ref). As a result, any property name starting with $ is
interpreted as metadata and cannot be deserialized properly.

### What is this change?
This update introduces custom logic in the converter’s Write and Read
methods to handle $-prefixed column names safely.

- During serialization, columns beginning with $ are escaped as "_$".

- During deserialization, this transformation is reversed to restore the
original property names.

### How was this tested

- [x] Unit tests

---------

Co-authored-by: Aniruddh Munde <[email protected]>

Author: Alekhya-Polavarapu

src/Core/Services/MetadataProviders/Converters/DatabaseObjectConverter.cs

@@ -17,6 +17,11 @@ namespace Azure.DataApiBuilder.Core.Services.MetadataProviders.Converters
     public class DatabaseObjectConverter : JsonConverter<DatabaseObject>
     {
         private const string TYPE_NAME = "TypeName";
+        private const string DOLLAR_CHAR = "$";
+
+        // ``DAB_ESCAPE$`` is used to escape column names that start with `$` during serialization.
+        // It is chosen to be unique enough to avoid collisions with actual column names.
+        private const string ESCAPED_DOLLARCHAR = "DAB_ESCAPE$";
 
         public override DatabaseObject Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
         {
@@ -29,6 +34,15 @@ public override DatabaseObject Read(ref Utf8JsonReader reader, Type typeToConver
 
                 DatabaseObject objA = (DatabaseObject)JsonSerializer.Deserialize(document, concreteType, options)!;
 
+                foreach (PropertyInfo prop in objA.GetType().GetProperties().Where(IsSourceDefinitionProperty))
+                {
+                    SourceDefinition? sourceDef = (SourceDefinition?)prop.GetValue(objA);
+                    if (sourceDef is not null)
+                    {
+                        UnescapeDollaredColumns(sourceDef);
+                    }
+                }
+
                 return objA;
             }
         }
@@ -58,12 +72,74 @@ public override void Write(Utf8JsonWriter writer, DatabaseObject value, JsonSeri
                 }
 
                 writer.WritePropertyName(prop.Name);
-                JsonSerializer.Serialize(writer, prop.GetValue(value), options);
+                object? propVal = prop.GetValue(value);
+                Type propType = prop.PropertyType;
+
+                // Only escape columns for properties whose type is exactly SourceDefinition (not subclasses).
+                // This is because, we do not want unnecessary mutation of subclasses of SourceDefinition unless needed.
+                if (IsSourceDefinitionProperty(prop) && propVal is SourceDefinition sourceDef && propVal.GetType() == typeof(SourceDefinition))
+                {
+                    EscapeDollaredColumns(sourceDef);
+                }
+
+                JsonSerializer.Serialize(writer, propVal, propType, options);
             }
 
             writer.WriteEndObject();
         }
 
+        private static bool IsSourceDefinitionProperty(PropertyInfo prop)
+        {
+            // Only return true for properties whose type is exactly SourceDefinition (not subclasses)
+            return prop.PropertyType == typeof(SourceDefinition);
+        }
+
+        /// <summary>
+        /// Escapes column keys that start with '$' to '_$' for serialization.
+        /// </summary>
+        private static void EscapeDollaredColumns(SourceDefinition sourceDef)
+        {
+            if (sourceDef.Columns is null || sourceDef.Columns.Count == 0)
+            {
+                return;
+            }
+
+            List<string> keysToEscape = sourceDef.Columns.Keys
+                .Where(k => k.StartsWith(DOLLAR_CHAR, StringComparison.Ordinal))
+                .ToList();
+
+            foreach (string key in keysToEscape)
+            {
+                ColumnDefinition col = sourceDef.Columns[key];
+                sourceDef.Columns.Remove(key);
+                string newKey = ESCAPED_DOLLARCHAR + key[1..];
+                sourceDef.Columns[newKey] = col;
+            }
+        }
+
+        /// <summary>
+        /// Unescapes column keys that start with '_$' to '$' for deserialization.
+        /// </summary>
+        private static void UnescapeDollaredColumns(SourceDefinition sourceDef)
+        {
+            if (sourceDef.Columns is null || sourceDef.Columns.Count == 0)
+            {
+                return;
+            }
+
+            List<string> keysToUnescape = sourceDef.Columns.Keys
+                .Where(k => k.StartsWith(ESCAPED_DOLLARCHAR, StringComparison.Ordinal))
+                .ToList();
+
+            foreach (string key in keysToUnescape)
+            {
+                ColumnDefinition col = sourceDef.Columns[key];
+                sourceDef.Columns.Remove(key);
+                string newKey = DOLLAR_CHAR + key[11..];
+                sourceDef.Columns[newKey] = col;
+            }
+        }
+
         private static Type GetTypeFromName(string typeName)
         {
             Type? type = Type.GetType(typeName);

src/Service.Tests/UnitTests/SerializationDeserializationTests.cs

@@ -277,8 +277,96 @@ public void TestDictionaryDatabaseObjectSerializationDeserialization()
             VerifySourceDefinitionSerializationDeserialization(deserializedDatabaseTable.TableDefinition, _databaseTable.TableDefinition, "FirstName");
         }
 
-        private void InitializeObjects()
+        /// <summary>
+        /// Validates serialization and deserilization of Dictionary containing DatabaseTable
+        /// The table will have dollar sign prefix ($) in the column name
+        /// this is how we serialize and deserialize metadataprovider.EntityToDatabaseObject dict.
+        /// </summary>
+        [TestMethod]
+        public void TestDictionaryDatabaseObjectSerializationDeserialization_WithDollarColumn()
+        {
+            InitializeObjects(generateDollaredColumn: true);
+
+            _options = new()
+            {
+                Converters = {
+                    new DatabaseObjectConverter(),
+                    new TypeConverter()
+                },
+                ReferenceHandler = ReferenceHandler.Preserve
+            };
+
+            Dictionary<string, DatabaseObject> dict = new() { { "person", _databaseTable } };
+
+            string serializedDict = JsonSerializer.Serialize(dict, _options);
+            Dictionary<string, DatabaseObject> deserializedDict = JsonSerializer.Deserialize<Dictionary<string, DatabaseObject>>(serializedDict, _options)!;
+
+            DatabaseTable deserializedDatabaseTable = (DatabaseTable)deserializedDict["person"];
+
+            Assert.AreEqual(deserializedDatabaseTable.SourceType, _databaseTable.SourceType);
+            Assert.AreEqual(deserializedDatabaseTable.FullName, _databaseTable.FullName);
+            deserializedDatabaseTable.Equals(_databaseTable);
+            VerifySourceDefinitionSerializationDeserialization(deserializedDatabaseTable.SourceDefinition, _databaseTable.SourceDefinition, "$FirstName");
+            VerifySourceDefinitionSerializationDeserialization(deserializedDatabaseTable.TableDefinition, _databaseTable.TableDefinition, "$FirstName");
+        }
+
+        /// <summary>
+        /// Validates serialization and deserilization of Dictionary containing DatabaseView
+        /// The table will have dollar sign prefix ($) in the column name
+        /// this is how we serialize and deserialize metadataprovider.EntityToDatabaseObject dict.
+        /// </summary>
+        [TestMethod]
+        public void TestDatabaseViewSerializationDeserialization_WithDollarColumn()
+        {
+            InitializeObjects(generateDollaredColumn: true);
+
+            TestTypeNameChanges(_databaseView, "DatabaseView");
+
+            // Test to catch if there is change in number of properties/fields
+            // Note: On Addition of property make sure it is added in following object creation _databaseView and include in serialization
+            // and deserialization test.
+            int fields = typeof(DatabaseView).GetProperties(BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance).Length;
+            Assert.AreEqual(fields, 6);
+
+            string serializedDatabaseView = JsonSerializer.Serialize(_databaseView, _options);
+            DatabaseView deserializedDatabaseView = JsonSerializer.Deserialize<DatabaseView>(serializedDatabaseView, _options)!;
+
+            Assert.AreEqual(deserializedDatabaseView.SourceType, _databaseView.SourceType);
+            deserializedDatabaseView.Equals(_databaseView);
+            VerifySourceDefinitionSerializationDeserialization(deserializedDatabaseView.SourceDefinition, _databaseView.SourceDefinition, "$FirstName");
+            VerifySourceDefinitionSerializationDeserialization(deserializedDatabaseView.ViewDefinition, _databaseView.ViewDefinition, "$FirstName");
+        }
+
+        /// <summary>
+        /// Validates serialization and deserilization of Dictionary containing DatabaseStoredProcedure
+        /// The table will have dollar sign prefix ($) in the column name
+        /// this is how we serialize and deserialize metadataprovider.EntityToDatabaseObject dict.
+        /// </summary>
+        [TestMethod]
+        public void TestDatabaseStoredProcedureSerializationDeserialization_WithDollarColumn()
         {
+            InitializeObjects(generateDollaredColumn: true);
+
+            TestTypeNameChanges(_databaseStoredProcedure, "DatabaseStoredProcedure");
+
+            // Test to catch if there is change in number of properties/fields
+            // Note: On Addition of property make sure it is added in following object creation _databaseStoredProcedure and include in serialization
+            // and deserialization test.
+            int fields = typeof(DatabaseStoredProcedure).GetProperties(BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance).Length;
+            Assert.AreEqual(fields, 6);
+
+            string serializedDatabaseSP = JsonSerializer.Serialize(_databaseStoredProcedure, _options);
+            DatabaseStoredProcedure deserializedDatabaseSP = JsonSerializer.Deserialize<DatabaseStoredProcedure>(serializedDatabaseSP, _options)!;
+
+            Assert.AreEqual(deserializedDatabaseSP.SourceType, _databaseStoredProcedure.SourceType);
+            deserializedDatabaseSP.Equals(_databaseStoredProcedure);
+            VerifySourceDefinitionSerializationDeserialization(deserializedDatabaseSP.SourceDefinition, _databaseStoredProcedure.SourceDefinition, "$FirstName", true);
+            VerifySourceDefinitionSerializationDeserialization(deserializedDatabaseSP.StoredProcedureDefinition, _databaseStoredProcedure.StoredProcedureDefinition, "$FirstName", true);
+        }
+
+        private void InitializeObjects(bool generateDollaredColumn = false)
+        {
+            string columnName = generateDollaredColumn ? "$FirstName" : "FirstName";
             _options = new()
             {
                 // ObjectConverter behavior different in .NET8 most likely due to
@@ -290,10 +378,11 @@ private void InitializeObjects()
                     new DatabaseObjectConverter(),
                     new TypeConverter()
                 }
+
             };
 
             _columnDefinition = GetColumnDefinition(typeof(string), DbType.String, true, false, false, new string("John"), false);
-            _sourceDefinition = GetSourceDefinition(false, false, new List<string>() { "FirstName" }, _columnDefinition);
+            _sourceDefinition = GetSourceDefinition(false, false, new List<string>() { columnName }, _columnDefinition);
 
             _databaseTable = new DatabaseTable()
             {
@@ -312,10 +401,10 @@ private void InitializeObjects()
                 {
                     IsInsertDMLTriggerEnabled = false,
                     IsUpdateDMLTriggerEnabled = false,
-                    PrimaryKey = new List<string>() { "FirstName" },
+                    PrimaryKey = new List<string>() { columnName },
                 },
             };
-            _databaseView.ViewDefinition.Columns.Add("FirstName", _columnDefinition);
+            _databaseView.ViewDefinition.Columns.Add(columnName, _columnDefinition);
 
             _parameterDefinition = new()
             {
@@ -332,10 +421,10 @@ private void InitializeObjects()
                 SourceType = EntitySourceType.StoredProcedure,
                 StoredProcedureDefinition = new()
                 {
-                    PrimaryKey = new List<string>() { "FirstName" },
+                    PrimaryKey = new List<string>() { columnName },
                 }
             };
-            _databaseStoredProcedure.StoredProcedureDefinition.Columns.Add("FirstName", _columnDefinition);
+            _databaseStoredProcedure.StoredProcedureDefinition.Columns.Add(columnName, _columnDefinition);
             _databaseStoredProcedure.StoredProcedureDefinition.Parameters.Add("Id", _parameterDefinition);
         }