DGS-18985 Fix JSON Schema ref handling (#2339)

* DGS-18985 Fix JSON Schema ref handling

* Minor cleanup

* Minor cleanup

Author: rayokota

src/Confluent.SchemaRegistry.Serdes.Json/JsonSchemaResolver.cs

@@ -16,8 +16,12 @@
 
 using System;
 using System.Collections.Generic;
+using System.IO;
+using System.Text.RegularExpressions;
+using System.Threading;
 using System.Threading.Tasks;
 using NJsonSchema;
+using NJsonSchema.References;
 using Newtonsoft.Json.Linq;
 #if NET8_0_OR_GREATER
 using NewtonsoftJsonSchemaGeneratorSettings = NJsonSchema.NewtonsoftJson.Generation.NewtonsoftJsonSchemaGeneratorSettings;
@@ -81,20 +85,17 @@ public async Task<JsonSchema> GetResolvedSchema(){
             return resolvedJsonSchema;
         }
 
-        private async Task CreateSchemaDictUtil(Schema root)
+        private async Task CreateSchemaDictUtil(Schema root, string referenceName = null)
         {
-            string rootStr = root.SchemaString;
-            JObject schema = JObject.Parse(rootStr);
-            string schemaId = (string)schema["$id"];
-            if (schemaId != null && !dictSchemaNameToSchema.ContainsKey(schemaId))
-                this.dictSchemaNameToSchema.Add(schemaId, root);
+            if (referenceName != null && !dictSchemaNameToSchema.ContainsKey(referenceName))
+                this.dictSchemaNameToSchema.Add(referenceName, root);
 
             if (root.References != null)
             {
                 foreach (var reference in root.References)
                 {
                     Schema refSchemaRes = await schemaRegistryClient.GetRegisteredSchemaAsync(reference.Subject, reference.Version, false);
-                    await CreateSchemaDictUtil(refSchemaRes);
+                    await CreateSchemaDictUtil(refSchemaRes, reference.Name);
                 }
             }
         }
@@ -118,21 +119,78 @@ private async Task<JsonSchema> GetSchemaUtil(Schema root)
                     new NJsonSchema.Generation.JsonSchemaResolver(rootObject, this.jsonSchemaGeneratorSettings ??
                         new NewtonsoftJsonSchemaGeneratorSettings());
 
-                JsonReferenceResolver referenceResolver =
-                    new JsonReferenceResolver(schemaResolver);
-                foreach (var reference in refers)
-                {
-                    JsonSchema jschema =
-                        dictSchemaNameToJsonSchema[reference.Name];
-                    referenceResolver.AddDocumentReference(reference.Name, jschema);
-                }
-                return referenceResolver;
+                return new CustomJsonReferenceResolver(schemaResolver, rootObject, dictSchemaNameToJsonSchema);
             };
 
             string rootStr = root.SchemaString;
             JObject schema = JObject.Parse(rootStr);
-            string schemaId = (string)schema["$id"];
+            string schemaId = (string)schema["$id"] ?? "";
             return await JsonSchema.FromJsonAsync(rootStr, schemaId, factory);
         }
+
+        private class CustomJsonReferenceResolver : JsonReferenceResolver
+        {
+            private JsonSchema rootObject;
+            private Dictionary<string, JsonSchema> refs;
+
+            public CustomJsonReferenceResolver(JsonSchemaAppender schemaAppender,
+                JsonSchema rootObject, Dictionary<string, JsonSchema> refs)
+                : base( schemaAppender)
+            {
+                this.rootObject = rootObject;
+                this.refs = refs;
+            }
+
+            public override string ResolveFilePath(string documentPath, string jsonPath)
+            {
+                // override the default behavior to not prepend the documentPath
+                var arr = Regex.Split(jsonPath, @"(?=#)");
+                return arr[0];
+            }
+
+            public override async Task<IJsonReference> ResolveFileReferenceAsync(string filePath, CancellationToken cancellationToken = default)
+            {
+                JsonSchema schema;
+                if (refs.TryGetValue(filePath, out schema))
+                {
+                    return schema;
+                }
+
+                // remove the documentPath and look for the reference
+                var fileName = Path.GetFileName(filePath);
+                if (refs.TryGetValue(fileName, out schema))
+                {
+                    return schema;
+                }
+
+                return await base.ResolveFileReferenceAsync(filePath, cancellationToken);
+            }
+
+            public override async Task<IJsonReference> ResolveUrlReferenceAsync(string url, CancellationToken cancellationToken = default)
+            {
+                JsonSchema schema;
+                if (refs.TryGetValue(url, out schema))
+                {
+                    return schema;
+                }
+
+                var documentPathProvider = rootObject as IDocumentPathProvider;
+                var documentPath = documentPathProvider?.DocumentPath;
+                if (documentPath != null)
+                {
+                    var documentUri = new Uri(documentPath);
+                    var uri = new Uri(url);
+                    var relativeUrl = documentUri.MakeRelativeUri(uri);
+
+                    // remove the documentPath and look for the reference
+                    if (refs.TryGetValue(relativeUrl.ToString(), out schema))
+                    {
+                        return schema;
+                    }
+                }
+
+                return await base.ResolveUrlReferenceAsync(url, cancellationToken);
+            }
+        }
     }
 }

test/Confluent.SchemaRegistry.Serdes.UnitTests/JsonSerializeDeserialize.cs

@@ -71,6 +71,36 @@ public class JsonSerializeDeserializeTests : BaseSerializeDeserializeTests
     }
   }
 }
+";
+        public string schema1NoId = @"
+{
+  ""$schema"": ""http://json-schema.org/draft-07/schema#"",
+  ""title"": ""Schema1"",
+  ""type"": ""object"",
+  ""properties"": {
+    ""field1"": {
+      ""type"": ""string""
+    },
+    ""field2"": {
+      ""type"": ""integer""
+    },
+    ""field3"": {
+      ""$ref"": ""http://schema2.json#/definitions/field""
+    }
+  }
+}
+";
+        public string schema2NoId = @"
+{
+  ""$schema"": ""http://json-schema.org/draft-07/schema#"",
+  ""title"": ""Schema2"",
+  ""type"": ""object"",
+  ""definitions"": {
+    ""field"": {
+      ""type"": ""boolean""
+    }
+  }
+}
 ";
         public class Schema1
         {
@@ -262,6 +292,57 @@ public async Task WithJsonSchemaExternalReferencesAsync()
             Assert.Equal(v.Field3, actual.Field3);
         }
 
+        [Fact]
+        public async Task WithJsonSchemaExternalReferencesNoIdAsync()
+        {
+            var subject1 = $"{testTopic}-Schema1";
+            var subject2 = $"{testTopic}-Schema2";
+
+            var registeredSchema2 = new RegisteredSchema(subject2, 1, 1, schema2NoId, SchemaType.Json, null);
+            store[schema2NoId] = 1;
+            subjectStore[subject2] = new List<RegisteredSchema> { registeredSchema2 };
+
+            var refs = new List<SchemaReference> { new SchemaReference("http://schema2.json", subject2, 1) };
+            var registeredSchema1 = new RegisteredSchema(subject1, 1, 2, schema1NoId, SchemaType.Json, refs);
+            store[schema1NoId] = 2;
+            subjectStore[subject1] = new List<RegisteredSchema> { registeredSchema1 };
+
+            var jsonSerializerConfig = new JsonSerializerConfig
+            {
+                UseLatestVersion = true,
+                AutoRegisterSchemas = false,
+                SubjectNameStrategy = SubjectNameStrategy.TopicRecord
+            };
+
+            var jsonSchemaGeneratorSettings = new NewtonsoftJsonSchemaGeneratorSettings
+            {
+                SerializerSettings = new JsonSerializerSettings
+                {
+                    ContractResolver = new DefaultContractResolver
+                    {
+                        NamingStrategy = new CamelCaseNamingStrategy()
+                    }
+                }
+            };
+
+            var jsonSerializer = new JsonSerializer<Schema1>(schemaRegistryClient, registeredSchema1,
+                jsonSerializerConfig, jsonSchemaGeneratorSettings);
+            var jsonDeserializer = new JsonDeserializer<Schema1>(schemaRegistryClient, registeredSchema1);
+            var v = new Schema1
+            {
+                Field1 = "Hello",
+                Field2 = 123,
+                Field3 = true
+            };
+            string expectedJson = "{\"field1\":\"Hello\",\"field2\":123,\"field3\":true}";
+            var bytes = await jsonSerializer.SerializeAsync(v, new SerializationContext(MessageComponentType.Value, testTopic));
+            Assert.NotNull(bytes);
+            Assert.Equal(expectedJson, Encoding.UTF8.GetString(bytes.AsSpan().Slice(5)));
+
+            var actual = await jsonDeserializer.DeserializeAsync(bytes, false, new SerializationContext(MessageComponentType.Value, testTopic));
+            Assert.Equal(v.Field3, actual.Field3);
+        }
+
 #if NET8_0_OR_GREATER
         [Theory]
         [InlineData("CamelCaseString", EnumType.EnumValue, "{\"Value\":\"enumValue\"}")]