Skip to content

Commit 3caa8a6

Browse files
sfc-gh-dstempniaksfc-gh-jszczerbinskisfc-gh-pbulawa
authored
SNOW-1729580 + SNOW-1739682 Remove encryption storage information in logs, remove retry on 400 (#882)
Co-authored-by: Jakub Szczerbiński <jakub.szczerbinski@snowflake.com> Co-authored-by: Piotr Bulawa <piotr.bulawa@snowflake.com>
1 parent e090783 commit 3caa8a6

File tree

5 files changed

+88
-4
lines changed

5 files changed

+88
-4
lines changed

‎cpp/logger/SecretDetector.cpp‎

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,14 @@ namespace Client
1717

1818
boost::regex SecretDetector::PRIVATE_KEY_DATA_PATTERN = boost::regex("\"privateKeyData\": \"([A-Za-z0-9/+=\\\\n]{10,})\"", boost::regex::extended | boost::regex::icase);
1919

20-
boost::regex SecretDetector::CONNECTION_TOKEN_PATTERN = boost::regex("(token|assertion content)(['\"\\s:=]+)([A-Za-z0-9=/_+-]{8,})", boost::regex::icase);
20+
boost::regex SecretDetector::CONNECTION_TOKEN_PATTERN = boost::regex("(token|assertion content|queryStageMasterKey|aws_key_id|aws_secret_key|aws_token)(['\"\\s:=]+)([A-Za-z0-9=/_+-]{8,})", boost::regex::icase);
2121

2222
boost::regex SecretDetector::PASSWORD_PATTERN = boost::regex("(password|passcode|pwd)(['\"\\s:=]+)([A-Za-z0-9!\"#$%&'\\()*+,-./:;<=>?@\\[\\]^_`\\{|\\}~]{6,})", boost::regex::icase);
2323

24+
boost::regex SecretDetector::ENCRYPTION_CREDS_IN_JSON_PATTERN = boost::regex("\"(encryptionMaterial|creds)\"\\s*:\\s*\\{.*?\\}", boost::regex::icase);
25+
26+
boost::regex SecretDetector::TOKEN_IN_JSON_PATTERN = boost::regex("\"(mastertoken|token)\":(\\t|\\s+)\"[a-zA-Z0-9=/_+-:]+\"", boost::regex::icase);
27+
2428
std::string SecretDetector::maskAwsKeys(std::string text)
2529
{
2630
return boost::regex_replace(text, SecretDetector::AWS_KEY_PATTERN, "$1$2'****'");
@@ -56,6 +60,16 @@ namespace Client
5660
return boost::regex_replace(text, SecretDetector::PASSWORD_PATTERN, "$1$2****");
5761
}
5862

63+
std::string SecretDetector::maskEncryptioncCredsInJson(std::string text)
64+
{
65+
return boost::regex_replace(text, SecretDetector::ENCRYPTION_CREDS_IN_JSON_PATTERN, "\"$1\": ****");
66+
}
67+
68+
std::string SecretDetector::maskTokenInJson(std::string text)
69+
{
70+
return boost::regex_replace(text, SecretDetector::TOKEN_IN_JSON_PATTERN, "\"$1\": ****");
71+
}
72+
5973
std::string SecretDetector::maskSecrets(std::string text)
6074
{
6175
return SecretDetector::maskAwsKeys(
@@ -65,7 +79,11 @@ namespace Client
6579
SecretDetector::maskPrivateKeyData(
6680
SecretDetector::maskConnectionToken(
6781
SecretDetector::maskPassword(
68-
text
82+
SecretDetector::maskEncryptioncCredsInJson(
83+
SecretDetector::maskTokenInJson(
84+
text
85+
)
86+
)
6987
)
7088
)
7189
)

‎cpp/logger/SecretDetector.hpp‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,8 @@ class SecretDetector
2727
static boost::regex PRIVATE_KEY_DATA_PATTERN;
2828
static boost::regex CONNECTION_TOKEN_PATTERN;
2929
static boost::regex PASSWORD_PATTERN;
30+
static boost::regex ENCRYPTION_CREDS_IN_JSON_PATTERN;
31+
static boost::regex TOKEN_IN_JSON_PATTERN;
3032

3133
static std::string maskAwsKeys(std::string text);
3234
static std::string maskAwsTokens(std::string text);
@@ -35,6 +37,8 @@ class SecretDetector
3537
static std::string maskPrivateKeyData(std::string text);
3638
static std::string maskConnectionToken(std::string text);
3739
static std::string maskPassword(std::string text);
40+
static std::string maskEncryptioncCredsInJson(std::string text);
41+
static std::string maskTokenInJson(std::string text);
3842
};
3943

4044
}

‎lib/connection.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -851,7 +851,7 @@ char_resp_cb(char *data, size_t size, size_t nmemb, RAW_CHAR_BUFFER *raw_buf) {
851851
}
852852

853853
sf_bool STDCALL is_retryable_http_code(long int code) {
854-
return ((code >= 500 && code < 600) || code == 400 || code == 403 ||
854+
return ((code >= 500 && code < 600) || code == 403 ||
855855
code == 408 || code == 429) ? SF_BOOLEAN_TRUE : SF_BOOLEAN_FALSE;
856856
}
857857

‎tests/test_unit_logger.c‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -502,6 +502,38 @@ void test_mask_secret_log() {
502502
"\"privateKeyData\": \"abcdefghijk\"",
503503
"\"privateKeyData\": \"XXXX\""
504504
},
505+
{//13
506+
"queryStageMasterKey: 123asdfasdfASDFasdf456asdfasdfASDFasdf==",
507+
"queryStageMasterKey: ****"
508+
},
509+
{//14
510+
"AWS_KEY_ID: AKIAIOSFODNN7EXAMPLE",
511+
"AWS_KEY_ID: ****"
512+
},
513+
{//15
514+
"AWS_SECRET_KEY: 123asdfasdfASDFasdf/456asdfasdfASDF/asdf",
515+
"AWS_SECRET_KEY: ****"
516+
},
517+
{//16
518+
"AWS_TOKEN: ETMsDgAAAXI0IS9NABRBRVMvQ0JDL1BLQ1M1UGFkZGluZwCAABAAEEb/xAQlmT+mwIx9G32E+ikAAACA/CPlEkq//+jWZnQkOj5VhjayruDsCVRGS/B6GzHUugXLc94EfEwuto94gS/oKSVrUg/JRPekypLAx4Afa1KW8n1RqXRF9Hzy1VVLmVEBMtei3yFJPNSHtfbeFHSr9eVB/OL8dOGbxQluGCh6XmaqTjyrh3fqUTWz7+n74+gu2ugAFFZ18iT+DStK0TTdmy4vBC6xUcHQ==",
519+
"AWS_TOKEN: ****"
520+
},
521+
{//17
522+
"\"encryptionMaterial\":\t{\n\t\t\t\"queryStageMasterKey\":\t\"123asdfasdfASDFasdf==\",\n\t\t\t\"queryId\":\t\"01b6f5ba-0002-0181-0000-11111111da\",\n\t\t\t\"smkId\":\t1111\n\t\t}",
523+
"\"encryptionMaterial\": ****"
524+
},
525+
{//18
526+
"\"creds\":\t{\n\t\t\t\t\"AWS_KEY_ID\":\t\"AKIAIOSFODNN7EXAMPLE\",\n\t\t\t\t\"AWS_SECRET_KEY\":\t\"123asdfasdfASDFasdf456asdfasdfASDFasdf\",\n\t\t\t\t\"AWS_TOKEN\":\t\"abc\",\n\t\t\t\t\"AWS_ID\":\t\"AKIAIOSFODNN7EXAMPLE\",\n\t\t\t\t\"AWS_KEY\":\t\"123asdfasdfASDFasdf456asdfasdfASDFasdf\"\n\t\t\t}",
527+
"\"creds\": ****"
528+
},
529+
{//19
530+
"\"token\":\t\"ETM:sDgAAA-XI0IS9NABRBRVMvQ0JDL1BLQ1M1UGFkZGluZwCAABAAEEb/xAQlmT+mwIx9G32E+ikAAACA/CPlEkq//+jWZnQkOj5VhjayruDsCVRGS/B6GzHUugXLc94EfEwuto94gS/oKSVrUg/JRPekypLAx4Afa1KW8n1RqXRF9Hzy1VVLmVEBMtei3yFJPNSHtfbeFHSr9eVB/OL8dOGbxQluGCh6XmaqTjyrh3fqUTWz7+n74+gu2ugAFFZ18iT+DStK0TTdmy4vBC6xUcHQ==\"",
531+
"\"token\": ****"
532+
},
533+
{//20
534+
"\"masterToken\":\t\"ETM:sDgAAA-XI0IS9NABRBRVMvQ0JDL1BLQ1M1UGFkZGluZwCAABAAEEb/xAQlmT+mwIx9G32E+ikAAACA/CPlEkq//+jWZnQkOj5VhjayruDsCVRGS/B6GzHUugXLc94EfEwuto94gS/oKSVrUg/JRPekypLAx4Afa1KW8n1RqXRF9Hzy1VVLmVEBMtei3yFJPNSHtfbeFHSr9eVB/OL8dOGbxQluGCh6XmaqTjyrh3fqUTWz7+n74+gu2ugAFFZ18iT+DStK0TTdmy4vBC6xUcHQ==\"",
535+
"\"masterToken\": ****"
536+
},
505537
};
506538

507539
char * line = NULL;

‎tests/test_unit_retry_context.c‎

Lines changed: 31 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
#define URL_AUTHENTICATOR "http://snowflake.com/session/authenticator-request"
2121

2222
void test_update_url_no_guid(void **unused) {
23+
SF_UNUSED(unused);
2324
char urlbuf[512];
2425
sf_sprintf(urlbuf, sizeof(urlbuf), "%s", URL_NO_GUID);
2526
RETRY_CONTEXT retry_ctx = {
@@ -36,6 +37,7 @@ void test_update_url_no_guid(void **unused) {
3637
}
3738

3839
void test_update_other_url_with_guid(void **unused) {
40+
SF_UNUSED(unused);
3941
char urlbuf[512];
4042
sf_sprintf(urlbuf, sizeof(urlbuf), "%s", URL_NON_QUERY_WITH_GUID);
4143
RETRY_CONTEXT retry_ctx = {
@@ -62,6 +64,7 @@ void test_update_other_url_with_guid(void **unused) {
6264
}
6365

6466
void test_update_query_url_with_retry_reason_disabled(void **unused) {
67+
SF_UNUSED(unused);
6568
char urlbuf[512];
6669
sf_sprintf(urlbuf, sizeof(urlbuf), "%s", URL_QUERY);
6770
RETRY_CONTEXT retry_ctx = {
@@ -138,6 +141,7 @@ void test_update_query_url_with_retry_reason_disabled(void **unused) {
138141
}
139142

140143
void test_update_query_url_with_retry_reason_enabled(void **unused) {
144+
SF_UNUSED(unused);
141145
char urlbuf[512];
142146
sf_sprintf(urlbuf, sizeof(urlbuf), "%s", URL_QUERY);
143147
RETRY_CONTEXT retry_ctx = {
@@ -205,6 +209,7 @@ void test_update_query_url_with_retry_reason_enabled(void **unused) {
205209
}
206210

207211
void test_new_retry_strategy(void **unused) {
212+
SF_UNUSED(unused);
208213
DECORRELATE_JITTER_BACKOFF djb = {
209214
SF_BACKOFF_BASE, //base
210215
SF_NEW_STRATEGY_BACKOFF_CAP //cap
@@ -218,7 +223,7 @@ void test_new_retry_strategy(void **unused) {
218223
sf_get_current_time_millis() // start time
219224
};
220225

221-
uint32 error_codes[SF_MAX_RETRY] = {429, 503, 403, 408, 400, 538, 525};
226+
uint32 error_codes[SF_MAX_RETRY] = {429, 503, 403, 503, 408, 538, 525};
222227
uint32 backoff = SF_BACKOFF_BASE;
223228
uint32 next_sleep_in_secs = 0;
224229
uint32 total_backoff = 0;
@@ -256,6 +261,7 @@ void test_new_retry_strategy(void **unused) {
256261
}
257262

258263
void test_retry_request_header(void **unused) {
264+
SF_UNUSED(unused);
259265
struct TESTCASE {
260266
const char* url;
261267
sf_bool has_app_header;
@@ -303,6 +309,29 @@ void test_retry_request_header(void **unused) {
303309
}
304310
}
305311

312+
void test_retryable_http_code(void **unused) {
313+
SF_UNUSED(unused);
314+
struct TEST_CODE {
315+
uint32 code;
316+
sf_bool retryable;
317+
};
318+
319+
struct TEST_CODE cases[] = {
320+
{ 400, SF_BOOLEAN_FALSE },
321+
{ 403, SF_BOOLEAN_TRUE },
322+
{ 404, SF_BOOLEAN_FALSE },
323+
{ 408, SF_BOOLEAN_TRUE },
324+
{ 429, SF_BOOLEAN_TRUE },
325+
{ 503, SF_BOOLEAN_TRUE },
326+
{ 600, SF_BOOLEAN_FALSE },
327+
};
328+
329+
for (unsigned i = 0; i < sizeof(cases) / sizeof(struct TEST_CODE); i++)
330+
{
331+
assert_int_equal(is_retryable_http_code(cases[i].code), cases[i].retryable);
332+
}
333+
}
334+
306335
int main(void) {
307336
const struct CMUnitTest tests[] = {
308337
cmocka_unit_test(test_update_url_no_guid),
@@ -311,6 +340,7 @@ int main(void) {
311340
cmocka_unit_test(test_update_query_url_with_retry_reason_enabled),
312341
cmocka_unit_test(test_new_retry_strategy),
313342
cmocka_unit_test(test_retry_request_header),
343+
cmocka_unit_test(test_retryable_http_code),
314344
};
315345
return cmocka_run_group_tests(tests, NULL, NULL);
316346
}

0 commit comments

Comments
 (0)