Skip to content

Commit 6e70fbe

Browse files
yinzarayinzara
authored
Fix ProtobufUtils Parse function (#2297) (#2298)
1 parent 00b1d01 commit 6e70fbe

File tree

4 files changed

+65
-56
lines changed

4 files changed

+65
-56
lines changed

‎src/Confluent.SchemaRegistry.Serdes.Protobuf/AssemblyInfo.cs‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
using System.Runtime.CompilerServices;
2+
3+
[assembly: InternalsVisibleTo("Confluent.SchemaRegistry.Serdes.UnitTests, " +
4+
"PublicKey=0024000004800000940000000602000000240000525341310004000001000100a9d95b0a1e" +
5+
"e3264dec3dff29931157b48768733b9ed4b1e8daba83d375872902c872ea4a79f389d51b574e000937c5" +
6+
"7d88952c128d4c156b8c2ac6fcd2a273e7ca3b2c0a29b5c30c81a9527f5fe7ef33d0a68040ae69f88c05" +
7+
"4181f81b1cbc2d429f0a054b1fe7d97bf32c6f781f2d483accf0faa54d1f502fad47744ddc4482")]

‎src/Confluent.SchemaRegistry.Serdes.Protobuf/ProtobufUtils.cs‎

Lines changed: 29 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
using System;
2020
using System.Collections;
2121
using System.Collections.Generic;
22+
using System.Collections.Immutable;
2223
using System.IO;
2324
using System.Linq;
2425
using System.Reflection;
@@ -36,11 +37,11 @@
3637
namespace Confluent.SchemaRegistry.Serdes
3738
{
3839
/// <summary>
39-
/// Protobuf utilities
40+
/// Protobuf utilities (internal utils for processing protobuf resources)
4041
/// </summary>
41-
public static class ProtobufUtils
42+
internal static class ProtobufUtils
4243
{
43-
public static IDictionary<string, string> BuiltIns = new Dictionary<string, string>
44+
private static IDictionary<string, string> BuiltIns = new Dictionary<string, string>
4445
{
4546
{ "confluent/meta.proto", GetResource("confluent.meta.proto") },
4647
{ "confluent/type/decimal.proto", GetResource("confluent.type.decimal.proto") },
@@ -68,8 +69,8 @@ public static class ProtobufUtils
6869
{ "google/protobuf/timestamp.proto", GetResource("google.protobuf.timestamp.proto") },
6970
{ "google/protobuf/type.proto", GetResource("google.protobuf.type.proto") },
7071
{ "google/protobuf/wrappers.proto", GetResource("google.protobuf.wrappers.proto") }
71-
};
72-
72+
}.ToImmutableDictionary();
73+
7374
private static string GetResource(string resourceName)
7475
{
7576
var info = Assembly.GetExecutingAssembly().GetName();
@@ -81,7 +82,7 @@ private static string GetResource(string resourceName)
8182
return streamReader.ReadToEnd();
8283
}
8384

84-
public static async Task<object> Transform(RuleContext ctx, object desc, object message,
85+
internal static async Task<object> Transform(RuleContext ctx, object desc, object message,
8586
IFieldTransform fieldTransform)
8687
{
8788
if (desc == null || message == null)
@@ -90,10 +91,10 @@ public static async Task<object> Transform(RuleContext ctx, object desc, object
9091
}
9192

9293
RuleContext.FieldContext fieldContext = ctx.CurrentField();
93-
94-
if (typeof(IList).IsAssignableFrom(message.GetType())
95-
|| (message.GetType().IsGenericType
96-
&& (message.GetType().GetGenericTypeDefinition() == typeof(List<>)
94+
95+
if (typeof(IList).IsAssignableFrom(message.GetType())
96+
|| (message.GetType().IsGenericType
97+
&& (message.GetType().GetGenericTypeDefinition() == typeof(List<>)
9798
|| message.GetType().GetGenericTypeDefinition() == typeof(IList<>))))
9899
{
99100
var tasks = ((IList<object>)message)
@@ -102,9 +103,9 @@ public static async Task<object> Transform(RuleContext ctx, object desc, object
102103
object[] items = await Task.WhenAll(tasks).ConfigureAwait(false);
103104
return items.ToList();
104105
}
105-
else if (typeof(IDictionary).IsAssignableFrom(message.GetType())
106-
|| (message.GetType().IsGenericType
107-
&& (message.GetType().GetGenericTypeDefinition() == typeof(Dictionary<,>)
106+
else if (typeof(IDictionary).IsAssignableFrom(message.GetType())
107+
|| (message.GetType().IsGenericType
108+
&& (message.GetType().GetGenericTypeDefinition() == typeof(Dictionary<,>)
108109
|| message.GetType().GetGenericTypeDefinition() == typeof(IDictionary<,>))))
109110
{
110111
return message;
@@ -156,7 +157,7 @@ public static async Task<object> Transform(RuleContext ctx, object desc, object
156157
ISet<string> ruleTags = ctx.Rule.Tags ?? new HashSet<string>();
157158
ISet<string> intersect = new HashSet<string>(fieldContext.Tags);
158159
intersect.IntersectWith(ruleTags);
159-
160+
160161
if (ruleTags.Count == 0 || intersect.Count != 0)
161162
{
162163
if (message is ByteString)
@@ -298,42 +299,36 @@ private static ISet<string> GetInlineTags(FieldDescriptorProto fd)
298299
return tags;
299300
}
300301

301-
public static FileDescriptorSet Parse(string schema, IDictionary<string, string> imports)
302+
internal static FileDescriptorSet Parse(string schema, IDictionary<string, string> imports)
302303
{
303-
IDictionary<string, string> allImports = new Dictionary<string, string>(BuiltIns);
304-
imports?.ToList().ForEach(x => allImports.Add(x.Key, x.Value));
305-
306-
var fds = new FileDescriptorSet();
307-
fds.FileSystem = new ProtobufImports(allImports);
308-
309-
fds.Add("__root.proto", true, new StringReader(schema));
310-
foreach (KeyValuePair<string, string> import in allImports)
304+
var fds = new FileDescriptorSet
311305
{
312-
fds.AddImportPath(import.Key);
313-
314-
}
306+
FileSystem = new ProtobufImports(imports)
307+
};
308+
fds.Add("__root.proto", true, new StringReader(schema));
309+
fds.AddImportPath(""); // all imports are relative in the filesystem so must make import path just empty string
315310
fds.Process();
316311
return fds;
317-
}
318-
319-
class ProtobufImports : IFileSystem
312+
}
313+
314+
private class ProtobufImports : IFileSystem
320315
{
321-
protected IDictionary<string, string> Imports { get; set; }
316+
private readonly IDictionary<string, string> imports;
322317

323318
public ProtobufImports(IDictionary<string, string> imports)
324319
{
325-
Imports = imports;
320+
this.imports = imports;
326321
}
327322

328323
public bool Exists(string path)
329324
{
330-
return Imports.ContainsKey(path);
325+
return BuiltIns.ContainsKey(path) || (imports?.ContainsKey(path) ?? false);
331326
}
332327

333328
public TextReader OpenText(string path)
334329
{
335-
return new StringReader(Imports[path]);
330+
return new StringReader(BuiltIns.TryGetValue(path, out var res) ? res : imports[path]);
336331
}
337332
}
338333
}
339-
}
334+
}

‎test/Confluent.SchemaRegistry.Serdes.UnitTests/Confluent.SchemaRegistry.Serdes.UnitTests.csproj‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@
66
<AssemblyName>Confluent.SchemaRegistry.Serdes.UnitTests</AssemblyName>
77
<TargetFrameworks>net6.0;net8.0</TargetFrameworks>
88
<GenerateRuntimeConfigurationFiles>true</GenerateRuntimeConfigurationFiles>
9+
<SignAssembly>true</SignAssembly>
10+
<AssemblyOriginatorKeyFile>..\..\src\Confluent.SchemaRegistry.Serdes.Protobuf\Confluent.SchemaRegistry.Serdes.Protobuf.snk</AssemblyOriginatorKeyFile>
911
</PropertyGroup>
1012

1113
<ItemGroup>

‎test/Confluent.SchemaRegistry.Serdes.UnitTests/ProtoSerializeDeserialize.cs‎

Lines changed: 27 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -46,11 +46,11 @@ public void ParseSchema()
4646
4747
message ReferrerMessage {
4848
49-
string root_id = 1 [(.confluent.field_meta) = { annotation: ""PII"" }];
50-
ReferencedMessage ref = 2 [(.confluent.field_meta).annotation = ""PII""];
49+
string root_id = 1 [(confluent.field_meta) = { doc: ""PII"" }];
50+
ReferencedMessage ref = 2 [(confluent.field_meta).doc = ""PII""];
5151
5252
}";
53-
53+
5454
string import = @"syntax = ""proto3"";
5555
package io.confluent.kafka.serializers.protobuf.test;
5656
@@ -62,15 +62,20 @@ message ReferencedMessage {
6262

6363
IDictionary<string, string> imports = new Dictionary<string, string>();
6464
imports["ref.proto"] = import;
65+
imports["confluent/meta.proto"] = "doesn't matter, will be overwritten anyway";
6566

6667
var fds = ProtobufUtils.Parse(schema, imports);
67-
foreach (var file in fds.Files)
68-
{
69-
foreach (var messageType in file.MessageTypes)
70-
{
71-
Assert.Equal("ReferrerMessage", messageType.Name);
72-
}
73-
}
68+
Assert.Equal(4, fds.Files.Count);
69+
70+
var fileNames = fds.Files.Select(s => s.Name).ToHashSet();
71+
Assert.Contains("__root.proto", fileNames);
72+
Assert.Contains("ref.proto", fileNames);
73+
Assert.Contains("confluent/meta.proto", fileNames);
74+
Assert.Contains("google/protobuf/descriptor.proto", fileNames);
75+
76+
var rootFile = fds.Files.First(s => s.Name == "__root.proto");
77+
Assert.Equal(1, rootFile.MessageTypes.Count);
78+
Assert.Equal("ReferrerMessage", rootFile.MessageTypes.First().Name);
7479
}
7580

7681
[Fact]
@@ -112,12 +117,12 @@ message Person {
112117
schema.RuleSet = new RuleSet(new List<Rule>(),
113118
new List<Rule>
114119
{
115-
new Rule("testCEL", RuleKind.Condition, RuleMode.Write, "CEL", null, null,
120+
new Rule("testCEL", RuleKind.Condition, RuleMode.Write, "CEL", null, null,
116121
"message.name == 'awesome'", null, null, false)
117122
}
118123
);
119124
store[schemaStr] = 1;
120-
subjectStore["topic-value"] = new List<RegisteredSchema> { schema };
125+
subjectStore["topic-value"] = new List<RegisteredSchema> { schema };
121126
var config = new ProtobufSerializerConfig
122127
{
123128
AutoRegisterSchemas = false,
@@ -159,12 +164,12 @@ message Person {
159164
schema.RuleSet = new RuleSet(new List<Rule>(),
160165
new List<Rule>
161166
{
162-
new Rule("testCEL", RuleKind.Condition, RuleMode.Write, "CEL", null, null,
167+
new Rule("testCEL", RuleKind.Condition, RuleMode.Write, "CEL", null, null,
163168
"message.name != 'awesome'", null, null, false)
164169
}
165170
);
166171
store[schemaStr] = 1;
167-
subjectStore["topic-value"] = new List<RegisteredSchema> { schema };
172+
subjectStore["topic-value"] = new List<RegisteredSchema> { schema };
168173
var config = new ProtobufSerializerConfig
169174
{
170175
AutoRegisterSchemas = false,
@@ -200,12 +205,12 @@ message Person {
200205
schema.RuleSet = new RuleSet(new List<Rule>(),
201206
new List<Rule>
202207
{
203-
new Rule("testCEL", RuleKind.Transform, RuleMode.Write, "CEL_FIELD", null, null,
208+
new Rule("testCEL", RuleKind.Transform, RuleMode.Write, "CEL_FIELD", null, null,
204209
"typeName == 'STRING' ; value + '-suffix'", null, null, false)
205210
}
206211
);
207212
store[schemaStr] = 1;
208-
subjectStore["topic-value"] = new List<RegisteredSchema> { schema };
213+
subjectStore["topic-value"] = new List<RegisteredSchema> { schema };
209214
var config = new ProtobufSerializerConfig
210215
{
211216
AutoRegisterSchemas = false,
@@ -247,12 +252,12 @@ message Person {
247252
schema.RuleSet = new RuleSet(new List<Rule>(),
248253
new List<Rule>
249254
{
250-
new Rule("testCEL", RuleKind.Condition, RuleMode.Write, "CEL_FIELD", null, null,
255+
new Rule("testCEL", RuleKind.Condition, RuleMode.Write, "CEL_FIELD", null, null,
251256
"name == 'name' ; value == 'awesome'", null, null, false)
252257
}
253258
);
254259
store[schemaStr] = 1;
255-
subjectStore["topic-value"] = new List<RegisteredSchema> { schema };
260+
subjectStore["topic-value"] = new List<RegisteredSchema> { schema };
256261
var config = new ProtobufSerializerConfig
257262
{
258263
AutoRegisterSchemas = false,
@@ -294,12 +299,12 @@ message Person {
294299
schema.RuleSet = new RuleSet(new List<Rule>(),
295300
new List<Rule>
296301
{
297-
new Rule("testCEL", RuleKind.Condition, RuleMode.Write, "CEL_FIELD", null, null,
302+
new Rule("testCEL", RuleKind.Condition, RuleMode.Write, "CEL_FIELD", null, null,
298303
"name == 'name' ; value != 'awesome'", null, null, false)
299304
}
300305
);
301306
store[schemaStr] = 1;
302-
subjectStore["topic-value"] = new List<RegisteredSchema> { schema };
307+
subjectStore["topic-value"] = new List<RegisteredSchema> { schema };
303308
var config = new ProtobufSerializerConfig
304309
{
305310
AutoRegisterSchemas = false,
@@ -332,7 +337,7 @@ message PersonWithPic {
332337
string name = 3 [(.confluent.field_meta) = { tags: ""PII"" }];
333338
bytes picture = 4 [(.confluent.field_meta) = { tags: ""PII"" }];
334339
}";
335-
340+
336341
var schema = new RegisteredSchema("topic-value", 1, 1, schemaStr, SchemaType.Protobuf, null);
337342
schema.Metadata = new Metadata(new Dictionary<string, ISet<string>>
338343
{
@@ -356,7 +361,7 @@ message PersonWithPic {
356361
}
357362
);
358363
store[schemaStr] = 1;
359-
subjectStore["topic-value"] = new List<RegisteredSchema> { schema };
364+
subjectStore["topic-value"] = new List<RegisteredSchema> { schema };
360365
var config = new ProtobufSerializerConfig
361366
{
362367
AutoRegisterSchemas = false,

0 commit comments

Comments
 (0)