Additional allocation hardening (#137)

* Based on additional fixes made when porting previous fix to 1.9 branch
  (#135)
* Check for failure in CRYPTO_THREAD_lock_new
* Reduce scope for unexpected behavior in RSA keygen

Author: samuel-lee-msft

KeysInUse/keysinuse.c

@@ -241,6 +241,10 @@ static void keysinuse_init_internal()
 
     lh_keysinuse_ctx_imp_lock = CRYPTO_THREAD_lock_new();
     lh_keysinuse_ctx_imp = lh_SCOSSL_KEYSINUSE_CTX_IMP_new(scossl_keysinuse_ctx_hash, scossl_keysinuse_ctx_cmp);
+    if (lh_keysinuse_ctx_imp_lock == NULL || lh_keysinuse_ctx_imp == NULL)
+    {
+        goto cleanup;
+    }
 
 #ifndef KEYSINUSE_LOG_SYSLOG
     if (!keysinuse_init_logging())

ScosslCommon/src/scossl_helpers.c

@@ -198,6 +198,11 @@ void SCOSSL_set_trace_level(int trace_level, int ossl_ERR_level)
 
 void SCOSSL_set_trace_log_filename(const char *filename)
 {
+    if( _loggingLock == NULL )
+    {
+        return;
+    }
+
     if( _traceLogFilename )
     {
         OPENSSL_free(_traceLogFilename);
@@ -239,6 +244,11 @@ static void _scossl_log_bytes_valist(
     char errStringBuf[SCOSSL_TRACELOG_PARA_LENGTH];
     char paraBuf[SCOSSL_TRACELOG_PARA_LENGTH];
     char *trace_level_prefix = "";
+    
+    if( _loggingLock == NULL )
+    {
+        return;
+    }
 
     if( SYMCRYPT_MAX(_traceLogLevel, _osslERRLogLevel) < trace_level )
     {

SymCryptProvider/src/keymgmt/p_scossl_rsa_keymgmt.c

@@ -401,7 +401,7 @@ static SCOSSL_PROV_RSA_KEY_CTX *p_scossl_rsa_keygen(_In_ SCOSSL_RSA_KEYGEN_CTX *
     PUINT64 pPubExp64;
     UINT32 genFlags = SYMCRYPT_FLAG_RSAKEY_SIGN | SYMCRYPT_FLAG_RSAKEY_ENCRYPT;
 
-    keyCtx = OPENSSL_malloc(sizeof(SCOSSL_PROV_RSA_KEY_CTX));
+    keyCtx = OPENSSL_zalloc(sizeof(SCOSSL_PROV_RSA_KEY_CTX));
     if (keyCtx == NULL)
     {
         goto cleanup;
@@ -434,7 +434,6 @@ static SCOSSL_PROV_RSA_KEY_CTX *p_scossl_rsa_keygen(_In_ SCOSSL_RSA_KEYGEN_CTX *
         goto cleanup;
     }
 
-    keyCtx->initialized = TRUE;
     keyCtx->keyType = genCtx->keyType;
     keyCtx->pssRestrictions = genCtx->pssRestrictions;
     genCtx->pssRestrictions = NULL;
@@ -443,6 +442,8 @@ static SCOSSL_PROV_RSA_KEY_CTX *p_scossl_rsa_keygen(_In_ SCOSSL_RSA_KEYGEN_CTX *
     keyCtx->keysinuseCtx = NULL;
 #endif
 
+    keyCtx->initialized = TRUE;
+
 cleanup:
     if (keyCtx != NULL && !keyCtx->initialized)
     {