Making more of our manual test cases automated, particularly around ssl certificate validation.

Author: stgates

Release/tests/Functional/http/client/authentication_tests.cpp

@@ -50,7 +50,7 @@ namespace tests { namespace functional { namespace http { namespace client {
 SUITE(authentication_tests)
 {
 
-TEST_FIXTURE(uri_address, auth_no_data, "Ignore:Linux", "698420", "Ignore:Apple", "698420")
+TEST_FIXTURE(uri_address, auth_no_data, "Ignore:Linux", "89", "Ignore:Apple", "89")
 {
     test_http_server::scoped_server scoped(m_uri);
     http_client_config client_config;
@@ -89,7 +89,7 @@ TEST_FIXTURE(uri_address, auth_no_data, "Ignore:Linux", "698420", "Ignore:Apple"
 
 // TFS 648783
 #ifndef __cplusplus_winrt
-TEST_FIXTURE(uri_address, proxy_auth_known_contentlength, "Ignore:Linux", "646268", "Ignore:Apple", "646268")
+TEST_FIXTURE(uri_address, proxy_auth_known_contentlength, "Ignore:Linux", "88", "Ignore:Apple", "88")
 {
     test_http_server::scoped_server scoped(m_uri);
     http_client_config client_config;
@@ -130,9 +130,7 @@ TEST_FIXTURE(uri_address, proxy_auth_known_contentlength, "Ignore:Linux", "64626
 }
 #endif
 
-TEST_FIXTURE(uri_address, proxy_auth_noseek,
-             "Ignore:Linux", "646268",
-             "Ignore:Apple", "646268")
+TEST_FIXTURE(uri_address, proxy_auth_noseek, "Ignore:Linux", "88", "Ignore:Apple", "88")
 {
     web::http::uri uri(U("http://localhost:34567/"));
     test_http_server::scoped_server scoped(uri);
@@ -164,9 +162,7 @@ TEST_FIXTURE(uri_address, proxy_auth_noseek,
 
 // Must specify content length with winrt client, so this test case isn't possible.
 #ifndef __cplusplus_winrt
-TEST_FIXTURE(uri_address, proxy_auth_unknown_contentlength, 
-            "Ignore:Linux", "646268",
-            "Ignore:Apple", "646268")
+TEST_FIXTURE(uri_address, proxy_auth_unknown_contentlength, "Ignore:Linux", "88", "Ignore:Apple", "88")
 {
     test_http_server::scoped_server scoped(m_uri);
     http_client_config client_config;
@@ -237,7 +233,7 @@ TEST_FIXTURE(uri_address, empty_username_password)
 // Fails on WinRT due to TFS 648278
 // Accessing a server that supports auth, but returns 401, even after the user has provided valid creds
 // We're making sure the error is reported properly, and the response data from the second response is received
-TEST_FIXTURE(uri_address, error_after_valid_credentials, "Ignore:Linux", "646268", "Ignore:Apple", "646268")
+TEST_FIXTURE(uri_address, error_after_valid_credentials, "Ignore:Linux", "89", "Ignore:Apple", "89")
 {
     web::http::uri uri(U("http://localhost:34569/"));
     test_http_server::scoped_server scoped(uri);
@@ -398,6 +394,7 @@ TEST_FIXTURE(server_properties, successful_auth_with_domain_cred, "Requires", "S
     VERIFY_ARE_EQUAL(status_codes::OK, response.status_code());
 }
 
+#ifndef __cplusplus_winrt // WinRT implementation doesn't support request buffer caching.
 TEST_FIXTURE(server_properties, failed_authentication_resend_request_error, "Requires", "Server;UserName;Password")
 {
     load_parameters();
@@ -414,14 +411,11 @@ TEST_FIXTURE(server_properties, failed_authentication_resend_request_error, "Req
     http_request request;
     request.set_method(methods::POST);
     request.set_body(data);
-    auto responseTask = client.request(request);
-
-    http_response response;
-
-    response = responseTask.get();
+    http_response response = client.request(request).get();
 
     VERIFY_ARE_EQUAL(200, response.status_code());
 }
+#endif
 
 #ifdef __cplusplus_winrt
 TEST_FIXTURE(uri_address, set_user_options_winrt)
@@ -604,7 +598,7 @@ TEST_FIXTURE(uri_address, set_user_options_exceptions)
 #pragma endregion 
 
 // Fix for 522831 AV after failed authentication attempt
-TEST_FIXTURE(uri_address, failed_authentication_attempt, "Ignore:Linux", "698420", "Ignore:Apple", "698420")
+TEST_FIXTURE(uri_address, failed_authentication_attempt, "Ignore:Linux", "89", "Ignore:Apple", "89")
 {
     http_client_config config;
     credentials cred(U("user"),U("schmuser"));

Release/tests/Functional/http/client/connections_and_errors.cpp

@@ -165,28 +165,6 @@ TEST_FIXTURE(uri_address, handshake_fail)
     VERIFY_THROWS(request.get(), http_exception);
 }
 
-#if !defined(__cplusplus_winrt)
-// Test case needs a server with invalid certificate running.
-TEST(server_cert_invalid, "Ignore", "Manual")
-{
-    http_client client(U("https://localhost"));
-
-    auto requestTask = client.request(methods::GET);
-    VERIFY_THROWS(requestTask.get(), http_exception);
-}
-
-// Test case needs a server with invalid certificate running.
-TEST(ignore_server_cert_invalid, "Ignore", "Manual")
-{
-    http_client_config config;
-    config.set_validate_certificates(false);
-    http_client client(U("https://localhost"), config);
-
-    auto request = client.request(methods::GET).get();
-    VERIFY_ARE_EQUAL(status_codes::OK, request.status_code());
-}
-#endif
-
 #if !defined(__cplusplus_winrt)
 // This test still sometimes segfaults on Linux, but I'm re-enabling it [AL]
 TEST_FIXTURE(uri_address, content_ready_timeout)
@@ -219,7 +197,7 @@ TEST_FIXTURE(uri_address, content_ready_timeout)
     listener.close().wait();
 }
 
-TEST_FIXTURE(uri_address, stream_timeout, "Ignore:Apple", "Bug 885080")
+TEST_FIXTURE(uri_address, stream_timeout, "Ignore:Apple", "149")
 {
     web::http::experimental::listener::http_listener listener(m_uri);
     listener.open().wait();
@@ -386,18 +364,6 @@ TEST_FIXTURE(uri_address, cancel_while_downloading_data)
 }
 #endif
 
-TEST_FIXTURE(uri_address, no_transfer_encoding_content_length, "Ignore", "Manual")
-{
-    http_client client(U("http://ws.audioscrobbler.com/2.0/?method=artist.gettoptracks&artist=cher&api_key=6fcd59047568e89b1615975081258990&format=json"));
-
-    client.request(methods::GET).then([](http_response response){
-        VERIFY_IS_FALSE(response.headers().has(header_names::content_length) 
-            && response.headers().has(header_names::transfer_encoding));
-        return response.extract_string();
-    }).then([](string_t result){
-        VERIFY_ARE_EQUAL(result.size(), 34686);  // hardcoded the content size here, need to check the correct number for manual test.
-    }).wait();
-}
 #pragma endregion
 
 } // SUITE(connections_and_errors)

Release/tests/Functional/http/client/outside_tests.cpp

@@ -40,8 +40,7 @@ namespace tests { namespace functional { namespace http { namespace client {
 SUITE(outside_tests)
 {
 
-TEST_FIXTURE(uri_address, outside_cnn_dot_com,
-             "Ignore", "Manual")
+TEST_FIXTURE(uri_address, outside_cnn_dot_com)
 {
     http_client client(U("http://www.cnn.com"));
 
@@ -62,8 +61,7 @@ TEST_FIXTURE(uri_address, outside_cnn_dot_com,
 #endif
 }
 
-TEST_FIXTURE(uri_address, outside_google_dot_com,
-             "Ignore", "Manual")
+TEST_FIXTURE(uri_address, outside_google_dot_com)
 {
     http_client client(U("http://www.google.com"));
 
@@ -84,8 +82,7 @@ TEST_FIXTURE(uri_address, outside_google_dot_com,
 #endif
 }
 
-TEST_FIXTURE(uri_address, reading_google_stream,
-             "Ignore", "Manual")
+TEST_FIXTURE(uri_address, reading_google_stream)
 {
     http_client simpleclient(U("http://www.google.com"));
     utility::string_t path = m_uri.query();
@@ -100,8 +97,62 @@ TEST_FIXTURE(uri_address, reading_google_stream,
     VERIFY_ARE_EQUAL(strcmp((const char *)chars, "<!doctype html><html itemscope=\"\" itemtype=\"http://schema.org/WebPage\""), 0);
 }
 
-TEST_FIXTURE(uri_address, outside_ssl_json,
-             "Ignore", "Manual")
+TEST_FIXTURE(uri_address, no_transfer_encoding_content_length)
+{
+    http_client client(U("http://ws.audioscrobbler.com/2.0/?method=artist.gettoptracks&artist=cher&api_key=6fcd59047568e89b1615975081258990&format=json"));
+
+    client.request(methods::GET).then([](http_response response){
+        VERIFY_ARE_EQUAL(response.status_code(), status_codes::OK);
+        VERIFY_IS_FALSE(response.headers().has(header_names::content_length)
+            && response.headers().has(header_names::transfer_encoding));
+        return response.extract_string();
+    }).then([](string_t result){
+        // Verify that the body size isn't empty.
+        VERIFY_IS_TRUE(result.size() > 0);
+    }).wait();
+}
+
+// Note additional sites for testing can be found at:
+// https://www.ssllabs.com/ssltest/
+// http://www.internetsociety.org/deploy360/resources/dane-test-sites/
+// https://onlinessl.netlock.hu/#
+TEST(server_selfsigned_cert, "Ignore:Android", "SSL certs not implemented")
+{
+    http_client client(U("https://www.pcwebshop.co.uk/"));
+    auto requestTask = client.request(methods::GET);
+    VERIFY_THROWS(requestTask.get(), http_exception);
+}
+
+TEST(server_hostname_mismatch, "Ignore:Android", "SSL certs not implemented")
+{
+    http_client client(U("https://swordsoftruth.com/"));
+    auto requestTask = client.request(methods::GET);
+    VERIFY_THROWS(requestTask.get(), http_exception);
+}
+
+TEST(server_cert_expired, "Ignore:Android", "SSL certs not implemented")
+{
+    http_client client(U("https://tv.eurosport.com/"));
+    auto requestTask = client.request(methods::GET);
+    VERIFY_THROWS(requestTask.get(), http_exception);
+}
+
+#if !defined(__cplusplus_winrt)
+TEST(ignore_server_cert_invalid, 
+     "Ignore:Android", "229",
+     "Ignore:Apple", "229",
+     "Ignore:Linux", "229")
+{
+    http_client_config config;
+    config.set_validate_certificates(false);
+    http_client client(U("https://www.pcwebshop.co.uk/"), config);
+
+    auto request = client.request(methods::GET).get();
+    VERIFY_ARE_EQUAL(status_codes::OK, request.status_code());
+}
+#endif
+
+TEST_FIXTURE(uri_address, outside_ssl_json, "Ignore:Android", "SSL certs not implemented")
 {
     // Create URI for:
     // https://www.googleapis.com/youtube/v3/playlistItems?part=snippet&playlistId=UUF1hMUVwlrvlVMjUGOZExgg&key=AIzaSyAviHxf_y0SzNoAq3iKqvWVE4KQ0yylsnk

Release/tests/Functional/http/client/proxy_tests.cpp

@@ -86,12 +86,8 @@ TEST_FIXTURE(uri_address, no_proxy_options_on_winrt)
 
     VERIFY_THROWS(client.request(methods::GET, U("/")).get(), http_exception);
 }
-#endif
 
-// Test is disabled since one time the office site was unavaliable.
-// We want 100% reliability in our automated test cases.
-TEST_FIXTURE(uri_address, proxy_with_credentials,
-             "Ignore", "Manual")
+TEST_FIXTURE(uri_address, proxy_with_credentials)
 {
     uri u(U("http://netproxy.redmond.corp.microsoft.com"));
 
@@ -112,6 +108,7 @@ TEST_FIXTURE(uri_address, proxy_with_credentials,
     VERIFY_ARE_EQUAL(status_codes::OK, response.status_code());
     response.content_ready().wait();
 }
+#endif
 
 } // SUITE(proxy_tests)
 

Release/tests/Functional/websockets/client/authentication_tests.cpp

@@ -84,7 +84,7 @@ TEST_FIXTURE(uri_address, auth_with_credentials, "Ignore:Linux", "NYI", "Ignore:
 }
 
 // Send and receive text message over SSL
-TEST_FIXTURE(uri_address, ssl_test, "Ignore", "Manual")
+TEST_FIXTURE(uri_address, ssl_test)
 {
     websocket_client client;
     client.connect(U("wss://echo.websocket.org/")).wait();