Skip to content

Commit 00b1d01

Browse files
rayokotarayokota
authored
Add CSFLE tests with pre-canned data (#22) (#2296)
* Add CSFLE tests with pre-canned data * Minor cleanup * Minor cleanup * Minor fix * Minor fix * Minor fix
1 parent bebc802 commit 00b1d01

File tree

3 files changed

+190
-2
lines changed

3 files changed

+190
-2
lines changed

‎src/Confluent.SchemaRegistry.Encryption.HcVault/HcVaultKmsClient.cs‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,12 +30,17 @@ public HcVaultKmsClient(string kekId, string ns, string tokenId)
3030
Namespace = ns;
3131
TokenId = tokenId;
3232

33-
if (!kekId.StartsWith(HcVaultKmsDriver.Prefix)) {
33+
if (!kekId.StartsWith(HcVaultKmsDriver.Prefix))
34+
{
3435
throw new ArgumentException(string.Format($"key URI must start with {HcVaultKmsDriver.Prefix}"));
3536
}
3637
keyId = KekId.Substring(HcVaultKmsDriver.Prefix.Length);
3738
IAuthMethodInfo authMethod = new TokenAuthMethodInfo(tokenId);
3839
Uri uri = new Uri(keyId);
40+
if (uri.Segments.Length == 0)
41+
{
42+
throw new ArgumentException(string.Format($"key URI must contain a key name"));
43+
}
3944
keyName = uri.Segments[^1];
4045

4146
var vaultClientSettings = new VaultClientSettings(uri.Scheme + "://" + uri.Authority, authMethod);

‎src/Confluent.SchemaRegistry.Encryption/LocalKmsClient.cs‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,12 @@ public LocalKmsClient(string secret)
2020
{
2121
secret = Environment.GetEnvironmentVariable("LOCAL_SECRET");
2222
}
23+
if (secret == null)
24+
{
25+
throw new ArgumentNullException("Cannot load secret");
26+
}
2327
Secret = secret;
24-
cryptor = new Cryptor(DekFormat.AES256_GCM);
28+
cryptor = new Cryptor(DekFormat.AES128_GCM);
2529
byte[] rawKey = Hkdf.DeriveKey(
2630
HashAlgorithmName.SHA256, Encoding.UTF8.GetBytes(secret), cryptor.KeySize());
2731
AesGcmKey aesGcm = new AesGcmKey();

‎test/Confluent.SchemaRegistry.Serdes.UnitTests/SerializeDeserialize.cs‎

Lines changed: 179 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -977,6 +977,185 @@ public void GenericRecordFieldEncryption()
977977
Assert.True(pic.SequenceEqual((byte[])result["picture"]));
978978
}
979979

980+
[Fact]
981+
public void GenericRecordFieldEncryptionF1Preserialized()
982+
{
983+
var schemaStr = "{\"type\":\"record\",\"name\":\"myrecord\"," +
984+
"\"fields\":[{\"name\":\"f1\",\"type\":\"string\"}]}";
985+
986+
var schema = new RegisteredSchema("topic-value", 1, 1, schemaStr, SchemaType.Avro, null);
987+
schema.Metadata = new Metadata(new Dictionary<string, ISet<string>>
988+
{
989+
["myrecord.f1"] = new HashSet<string> { "PII" },
990+
991+
}, new Dictionary<string, string>(), new HashSet<string>()
992+
);
993+
schema.RuleSet = new RuleSet(new List<Rule>(),
994+
new List<Rule>
995+
{
996+
new Rule("encryptPII", RuleKind.Transform, RuleMode.WriteRead, "ENCRYPT", new HashSet<string>
997+
{
998+
"PII"
999+
}, new Dictionary<string, string>
1000+
{
1001+
["encrypt.kek.name"] = "kek1",
1002+
["encrypt.kms.type"] = "local-kms",
1003+
["encrypt.kms.key.id"] = "mykey"
1004+
})
1005+
}
1006+
);
1007+
store[schemaStr] = 1;
1008+
subjectStore["topic-value"] = new List<RegisteredSchema> { schema };
1009+
var config = new AvroDeserializerConfig
1010+
{
1011+
};
1012+
config.Set("rules.secret", "mysecret");
1013+
IRuleExecutor ruleExecutor = new FieldEncryptionExecutor(dekRegistryClient, clock);
1014+
var deserializer = new AvroDeserializer<GenericRecord>(schemaRegistryClient, config, new List<IRuleExecutor>{ ruleExecutor});
1015+
1016+
kekStore[new KekId("kek1", false)] =
1017+
new RegisteredKek()
1018+
{
1019+
Name = "kek1",
1020+
KmsType = "local-kms",
1021+
KmsKeyId = "mykey",
1022+
};
1023+
dekStore[new DekId("kek1", "topic-value", 1, DekFormat.AES256_GCM, false)] =
1024+
new RegisteredDek()
1025+
{
1026+
Subject = "topic-value",
1027+
Version = 1,
1028+
Algorithm = DekFormat.AES256_GCM,
1029+
EncryptedKeyMaterial =
1030+
"07V2ndh02DA73p+dTybwZFm7DKQSZN1tEwQh+FoX1DZLk4Yj2LLu4omYjp/84tAg3BYlkfGSz+zZacJHIE4=",
1031+
};
1032+
Headers headers = new Headers();
1033+
byte[] bytes = new byte[]{0, 0, 0, 0, 1, 104, 122, 103, 121, 47, 106, 70, 78, 77, 86, 47, 101, 70, 105, 108, 97, 72, 114, 77, 121, 101, 66, 103, 100, 97, 86, 122, 114, 82, 48, 117, 100, 71, 101, 111, 116, 87, 56, 99, 65, 47, 74, 97, 108, 55, 117, 107, 114, 43, 77, 47, 121, 122};
1034+
var result = deserializer.DeserializeAsync(bytes, false, new SerializationContext(MessageComponentType.Value, testTopic, headers)).Result;
1035+
1036+
Assert.Equal("hello world", result["f1"]);
1037+
}
1038+
1039+
[Fact]
1040+
public void GenericRecordFieldEncryptionDeterministicF1Preserialized()
1041+
{
1042+
var schemaStr = "{\"type\":\"record\",\"name\":\"myrecord\"," +
1043+
"\"fields\":[{\"name\":\"f1\",\"type\":\"string\"}]}";
1044+
1045+
var schema = new RegisteredSchema("topic-value", 1, 1, schemaStr, SchemaType.Avro, null);
1046+
schema.Metadata = new Metadata(new Dictionary<string, ISet<string>>
1047+
{
1048+
["myrecord.f1"] = new HashSet<string> { "PII" },
1049+
1050+
}, new Dictionary<string, string>(), new HashSet<string>()
1051+
);
1052+
schema.RuleSet = new RuleSet(new List<Rule>(),
1053+
new List<Rule>
1054+
{
1055+
new Rule("encryptPII", RuleKind.Transform, RuleMode.WriteRead, "ENCRYPT", new HashSet<string>
1056+
{
1057+
"PII"
1058+
}, new Dictionary<string, string>
1059+
{
1060+
["encrypt.kek.name"] = "kek1",
1061+
["encrypt.kms.type"] = "local-kms",
1062+
["encrypt.kms.key.id"] = "mykey",
1063+
["encrypt.dek.algorithm"] = "AES256_SIV",
1064+
})
1065+
}
1066+
);
1067+
store[schemaStr] = 1;
1068+
subjectStore["topic-value"] = new List<RegisteredSchema> { schema };
1069+
var config = new AvroDeserializerConfig
1070+
{
1071+
};
1072+
config.Set("rules.secret", "mysecret");
1073+
IRuleExecutor ruleExecutor = new FieldEncryptionExecutor(dekRegistryClient, clock);
1074+
var deserializer = new AvroDeserializer<GenericRecord>(schemaRegistryClient, config, new List<IRuleExecutor>{ ruleExecutor});
1075+
1076+
kekStore[new KekId("kek1", false)] =
1077+
new RegisteredKek()
1078+
{
1079+
Name = "kek1",
1080+
KmsType = "local-kms",
1081+
KmsKeyId = "mykey",
1082+
};
1083+
dekStore[new DekId("kek1", "topic-value", 1, DekFormat.AES256_SIV, false)] =
1084+
new RegisteredDek()
1085+
{
1086+
Subject = "topic-value",
1087+
Version = 1,
1088+
Algorithm = DekFormat.AES256_SIV,
1089+
EncryptedKeyMaterial =
1090+
"YSx3DTlAHrmpoDChquJMifmPntBzxgRVdMzgYL82rgWBKn7aUSnG+WIu9ozBNS3y2vXd++mBtK07w4/W/G6w0da39X9hfOVZsGnkSvry/QRht84V8yz3dqKxGMOK5A==",
1091+
};
1092+
Headers headers = new Headers();
1093+
byte[] bytes = new byte[]{0, 0, 0, 0, 1, 72, 68, 54, 89, 116, 120, 114, 108, 66, 110, 107, 84, 87, 87, 57, 78, 54, 86, 98, 107, 51, 73, 73, 110, 106, 87, 72, 56, 49, 120, 109, 89, 104, 51, 107, 52, 100};
1094+
var result = deserializer.DeserializeAsync(bytes, false, new SerializationContext(MessageComponentType.Value, testTopic, headers)).Result;
1095+
1096+
Assert.Equal("hello world", result["f1"]);
1097+
}
1098+
1099+
[Fact]
1100+
public void GenericRecordFieldEncryptionDekRotationF1Preserialized()
1101+
{
1102+
var schemaStr = "{\"type\":\"record\",\"name\":\"myrecord\"," +
1103+
"\"fields\":[{\"name\":\"f1\",\"type\":\"string\"}]}";
1104+
1105+
var schema = new RegisteredSchema("topic-value", 1, 1, schemaStr, SchemaType.Avro, null);
1106+
schema.Metadata = new Metadata(new Dictionary<string, ISet<string>>
1107+
{
1108+
["myrecord.f1"] = new HashSet<string> { "PII" },
1109+
1110+
}, new Dictionary<string, string>(), new HashSet<string>()
1111+
);
1112+
schema.RuleSet = new RuleSet(new List<Rule>(),
1113+
new List<Rule>
1114+
{
1115+
new Rule("encryptPII", RuleKind.Transform, RuleMode.WriteRead, "ENCRYPT", new HashSet<string>
1116+
{
1117+
"PII"
1118+
}, new Dictionary<string, string>
1119+
{
1120+
["encrypt.kek.name"] = "kek1",
1121+
["encrypt.kms.type"] = "local-kms",
1122+
["encrypt.kms.key.id"] = "mykey",
1123+
["encrypt.dek.expiry.days"] = "1"
1124+
})
1125+
}
1126+
);
1127+
store[schemaStr] = 1;
1128+
subjectStore["topic-value"] = new List<RegisteredSchema> { schema };
1129+
var config = new AvroDeserializerConfig
1130+
{
1131+
};
1132+
config.Set("rules.secret", "mysecret");
1133+
IRuleExecutor ruleExecutor = new FieldEncryptionExecutor(dekRegistryClient, clock);
1134+
var deserializer = new AvroDeserializer<GenericRecord>(schemaRegistryClient, config, new List<IRuleExecutor>{ ruleExecutor});
1135+
1136+
kekStore[new KekId("kek1", false)] =
1137+
new RegisteredKek()
1138+
{
1139+
Name = "kek1",
1140+
KmsType = "local-kms",
1141+
KmsKeyId = "mykey",
1142+
};
1143+
dekStore[new DekId("kek1", "topic-value", 1, DekFormat.AES256_GCM, false)] =
1144+
new RegisteredDek()
1145+
{
1146+
Subject = "topic-value",
1147+
Version = 1,
1148+
Algorithm = DekFormat.AES256_GCM,
1149+
EncryptedKeyMaterial =
1150+
"W/v6hOQYq1idVAcs1pPWz9UUONMVZW4IrglTnG88TsWjeCjxmtRQ4VaNe/I5dCfm2zyY9Cu0nqdvqImtUk4=",
1151+
};
1152+
Headers headers = new Headers();
1153+
byte[] bytes = new byte[]{0, 0, 0, 0, 1, 120, 65, 65, 65, 65, 65, 65, 71, 52, 72, 73, 54, 98, 49, 110, 88, 80, 88, 113, 76, 121, 71, 56, 99, 73, 73, 51, 53, 78, 72, 81, 115, 101, 113, 113, 85, 67, 100, 43, 73, 101, 76, 101, 70, 86, 65, 101, 78, 112, 83, 83, 51, 102, 120, 80, 110, 74, 51, 50, 65, 61};
1154+
var result = deserializer.DeserializeAsync(bytes, false, new SerializationContext(MessageComponentType.Value, testTopic, headers)).Result;
1155+
1156+
Assert.Equal("hello world", result["f1"]);
1157+
}
1158+
9801159
[Fact]
9811160
public void GenericRecordJSONataFullyCompatible()
9821161
{

0 commit comments

Comments
 (0)