Skip to content

Commit 7fb2dc4

Browse files
sfc-gh-knozderkosfc-gh-knozderko
authored
SNOW-1887537 Get file without SFC_DIGEST (#1081)
1 parent ae9f693 commit 7fb2dc4

File tree

5 files changed

+125
-23
lines changed

5 files changed

+125
-23
lines changed

‎Snowflake.Data.Tests/UnitTests/SFAzureClientTest.cs‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -410,5 +410,64 @@ public void TestEncryptionMetadataReadingIsCaseInsensitive()
410410
Assert.AreEqual("key", fileHeader.encryptionMetadata.key);
411411
Assert.AreEqual("description", fileHeader.encryptionMetadata.matDesc);
412412
}
413+
414+
[Test]
415+
public void TestEncryptionMetadataReadingSucceedsWithoutSfcDigest()
416+
{
417+
// arrange
418+
var metadata = new Dictionary<string, string>
419+
{
420+
{
421+
"encryptiondata",
422+
@"{
423+
""ContentEncryptionIV"": ""initVector"",
424+
""WrappedContentKey"": {
425+
""EncryptedKey"": ""key""
426+
}
427+
}"
428+
},
429+
{ "matdesc", "description" }
430+
};
431+
var blobProperties = BlobsModelFactory.BlobProperties(metadata: metadata, contentLength: 10);
432+
var mockBlobServiceClient = new Mock<BlobServiceClient>();
433+
_client = new SFSnowflakeAzureClient(_fileMetadata.stageInfo, mockBlobServiceClient.Object);
434+
435+
// act
436+
var fileHeader = _client.HandleFileHeaderResponse(ref _fileMetadata, blobProperties);
437+
438+
// assert
439+
Assert.AreEqual(ResultStatus.UPLOADED.ToString(), _fileMetadata.resultStatus);
440+
Assert.IsNull(fileHeader.digest);
441+
Assert.AreEqual("initVector", fileHeader.encryptionMetadata.iv);
442+
Assert.AreEqual("key", fileHeader.encryptionMetadata.key);
443+
Assert.AreEqual("description", fileHeader.encryptionMetadata.matDesc);
444+
}
445+
446+
[Test]
447+
public void TestEncryptionMetadataReadingFailsWhenMandatoryPropertyIsMissing()
448+
{
449+
// arrange
450+
var metadataWithoutMatDesc = new Dictionary<string, string>
451+
{
452+
{
453+
"encryptiondata",
454+
@"{
455+
""ContentEncryptionIV"": ""initVector"",
456+
""WrappedContentKey"": {
457+
""EncryptedKey"": ""key""
458+
}
459+
}"
460+
}
461+
};
462+
var blobProperties = BlobsModelFactory.BlobProperties(metadata: metadataWithoutMatDesc, contentLength: 10);
463+
var mockBlobServiceClient = new Mock<BlobServiceClient>();
464+
_client = new SFSnowflakeAzureClient(_fileMetadata.stageInfo, mockBlobServiceClient.Object);
465+
466+
// act
467+
var thrown = Assert.Throws<KeyNotFoundException>(() => _client.HandleFileHeaderResponse(ref _fileMetadata, blobProperties));
468+
469+
// assert
470+
Assert.That(thrown.Message, Does.Contain("The given key 'matdesc' was not present in the dictionary."));
471+
}
413472
}
414473
}

‎Snowflake.Data.Tests/UnitTests/SFGCSClientTest.cs‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -410,6 +410,27 @@ public void TestGcsHeadersAreCaseInsensitiveForWebHeaderCollection(string header
410410
Assert.AreEqual(HeaderValue, header.First());
411411
}
412412

413+
[Test]
414+
public void TestHandleGetFileHeaderResponseWithoutSfcDigest()
415+
{
416+
// arrange
417+
var headers = new WebHeaderCollection();
418+
headers.Add("content-length", "123");
419+
var stageInfo = new PutGetStageInfo() { stageCredentials = new Dictionary<string, string>() };
420+
var client = new SFGCSClient(stageInfo);
421+
var response = new Mock<HttpWebResponse>();
422+
response.Setup(r => r.Headers).Returns(headers);
423+
var fileMetadata = new SFFileMetadata();
424+
425+
// act
426+
var fileHeader = client.handleGetFileHeaderResponse(response.Object, fileMetadata);
427+
428+
// assert
429+
Assert.AreEqual(ResultStatus.UPLOADED.ToString(), fileMetadata.resultStatus);
430+
Assert.IsNull(fileHeader.digest);
431+
Assert.AreEqual(123, fileHeader.contentLength);
432+
}
433+
413434
private void AssertForDownloadFileTests(ResultStatus expectedResultStatus)
414435
{
415436
if (expectedResultStatus == ResultStatus.DOWNLOADED)

‎Snowflake.Data.Tests/UnitTests/SFS3ClientTest.cs‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -341,6 +341,28 @@ public void TestEncryptionMetadataReadingIsCaseInsensitive()
341341
Assert.AreEqual("description", fileHeader.encryptionMetadata.matDesc);
342342
}
343343

344+
[Test]
345+
public void TestReadingMetadataSucceedsWithoutSfcDigest()
346+
{
347+
// arrange
348+
var mockAmazonS3Client = new Mock<AmazonS3Client>(AwsKeyId, AwsSecretKey, AwsToken, _clientConfig);
349+
_client = new SFS3Client(_fileMetadata.stageInfo, MaxRetry, Parallel, _proxyCredentials, mockAmazonS3Client.Object);
350+
var response = new GetObjectResponse();
351+
response.Metadata.Add(SFS3Client.AMZ_IV, "initVector");
352+
response.Metadata.Add(SFS3Client.AMZ_KEY, "key");
353+
response.Metadata.Add(SFS3Client.AMZ_MATDESC, "description");
354+
355+
// act
356+
var fileHeader = _client.HandleFileHeaderResponse(ref _fileMetadata, response);
357+
358+
// assert
359+
Assert.AreEqual(ResultStatus.UPLOADED.ToString(), _fileMetadata.resultStatus);
360+
Assert.IsNull(fileHeader.digest);
361+
Assert.AreEqual("initVector", fileHeader.encryptionMetadata.iv);
362+
Assert.AreEqual("key", fileHeader.encryptionMetadata.key);
363+
Assert.AreEqual("description", fileHeader.encryptionMetadata.matDesc);
364+
}
365+
344366
private void AssertForDownloadFileTests(ResultStatus expectedResultStatus)
345367
{
346368
if (expectedResultStatus == ResultStatus.DOWNLOADED)

‎Snowflake.Data/Core/FileTransfer/StorageClient/SFGCSClient.cs‎

Lines changed: 16 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -168,16 +168,7 @@ public FileHeader GetFileHeader(SFFileMetadata fileMetadata)
168168

169169
using (HttpWebResponse response = (HttpWebResponse)request.GetResponse())
170170
{
171-
var digest = response.Headers.GetValues(GCS_METADATA_SFC_DIGEST);
172-
var contentLength = response.Headers.GetValues("content-length");
173-
174-
fileMetadata.resultStatus = ResultStatus.UPLOADED.ToString();
175-
176-
return new FileHeader
177-
{
178-
digest = digest[0],
179-
contentLength = Convert.ToInt64(contentLength[0])
180-
};
171+
return handleGetFileHeaderResponse(response, fileMetadata);
181172
}
182173
}
183174
catch (WebException ex)
@@ -217,16 +208,7 @@ public async Task<FileHeader> GetFileHeaderAsync(SFFileMetadata fileMetadata, Ca
217208

218209
using (HttpWebResponse response = (HttpWebResponse)await request.GetResponseAsync().ConfigureAwait(false))
219210
{
220-
var digest = response.Headers.GetValues(GCS_METADATA_SFC_DIGEST);
221-
var contentLength = response.Headers.GetValues("content-length");
222-
223-
fileMetadata.resultStatus = ResultStatus.UPLOADED.ToString();
224-
225-
return new FileHeader
226-
{
227-
digest = digest[0],
228-
contentLength = Convert.ToInt64(contentLength[0])
229-
};
211+
return handleGetFileHeaderResponse(response, fileMetadata);
230212
}
231213
}
232214
catch (WebException ex)
@@ -240,6 +222,20 @@ public async Task<FileHeader> GetFileHeaderAsync(SFFileMetadata fileMetadata, Ca
240222
return null;
241223
}
242224

225+
internal FileHeader handleGetFileHeaderResponse(HttpWebResponse response, SFFileMetadata fileMetadata)
226+
{
227+
var digest = response.Headers.GetValues(GCS_METADATA_SFC_DIGEST);
228+
var contentLength = response.Headers.GetValues("content-length");
229+
230+
fileMetadata.resultStatus = ResultStatus.UPLOADED.ToString();
231+
232+
return new FileHeader
233+
{
234+
digest = digest?[0],
235+
contentLength = Convert.ToInt64(contentLength[0])
236+
};
237+
}
238+
243239
internal string generateFileURL(PutGetStageInfo stageInfo, string fileName)
244240
{
245241
var storageHostPath = ExtractStorageHostPath(stageInfo);

‎Snowflake.Data/Core/FileTransfer/StorageClient/SFSnowflakeAzureClient.cs‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -173,7 +173,7 @@ internal FileHeader HandleFileHeaderResponse(ref SFFileMetadata fileMetadata, Bl
173173

174174
return new FileHeader
175175
{
176-
digest = GetMetadataValueCaseInsensitive(response, "sfcdigest"),
176+
digest = GetMetadataValueCaseInsensitive(response, "sfcdigest", false),
177177
contentLength = response.ContentLength,
178178
encryptionMetadata = encryptionMetadata
179179
};
@@ -190,11 +190,15 @@ private bool TryGetMetadataValueCaseInsensitive(BlobProperties properties, strin
190190
return keysCaseInsensitive.Any() ? properties.Metadata.TryGetValue(keysCaseInsensitive.First(), out metadataValue) : false;
191191
}
192192

193-
private string GetMetadataValueCaseInsensitive(BlobProperties properties, string metadataKey)
193+
private string GetMetadataValueCaseInsensitive(BlobProperties properties, string metadataKey, bool failIfNotFound = true)
194194
{
195195
if (TryGetMetadataValueCaseInsensitive(properties, metadataKey, out var metadataValue))
196196
return metadataValue;
197-
throw new KeyNotFoundException($"The given key '{metadataKey}' was not present in the dictionary.");
197+
if (failIfNotFound)
198+
{
199+
throw new KeyNotFoundException($"The given key '{metadataKey}' was not present in the dictionary.");
200+
}
201+
return null;
198202
}
199203

200204
/// <summary>

0 commit comments

Comments
 (0)