Changes done for v1.0.8

Author: nysenthil

com.ibm.streamsx.websocket/CHANGELOG.md

@@ -1,5 +1,11 @@
 Changes
 =======
+## v1.0.8:
+* Sep/22/2020
+* Added logic to stop the Boost ASIO run loop on operator shutdown in order for an immediate closure of the TLS and non-TLS server ports in the Source and Sink operators.
+* Added an optional tlsCipherWhitelist parameter in the Source and Sink operators for the user to specify approved TLS/SSL ciphers thereby avoiding the use of any security vulnerable ciphers.
+* Added logic in the HttpPost operator to emit an output tuple even when the tuple processing logic encounters an exception.
+
 ## v1.0.7:
 * Sep/01/2020
 * Made the WebSocketSource operator friendlier to browser-based client applications so that they can do HTTP GET for fetching files such as html, css, js, png, gif, favicon etc.

com.ibm.streamsx.websocket/com.ibm.streamsx.websocket.op/WebSocketSink/WebSocketSink.xml

@@ -313,6 +313,16 @@
         <type>uint32</type>
         <cardinality>1</cardinality>
       </parameter>
+      
+      <parameter>
+        <name>tlsCipherWhitelist</name>
+        <description>This parameter can be used to specify a string containing one or more comma separated TLS/SSL ciphers that should be used during TLS/SSL connection negotiations with clients. It is handy when there is a need to avoid using ciphers that are found to have security vulnerabilities. (Default is an empty string.)</description>
+        <optional>true</optional>
+        <rewriteAllowed>true</rewriteAllowed>
+        <expressionMode>AttributeFree</expressionMode>
+        <type>rstring</type>
+        <cardinality>1</cardinality>
+      </parameter>
     </parameters>
 
     <inputPorts>

com.ibm.streamsx.websocket/com.ibm.streamsx.websocket.op/WebSocketSink/WebSocketSink_cpp.cgt

@@ -8,7 +8,7 @@
 /*
  ============================================================
  First created on: Apr/28/2020
- Last modified on: Aug/31/2020
+ Last modified on: Sep/22/2020
 
  This particular operator (WebSocketSink) is used to
  send text (plain text, JSON, XML) and/or binary data to
@@ -55,6 +55,9 @@
 #include <boost/exception/to_string.hpp>
 #include <boost/thread/thread.hpp>
 
+// Necessary headers for setting user specified TLS ciphers.
+#include <openssl/ssl.h>
+
 // For the core logic below, we will need these namespaces from websocketpp.
 using websocketpp::lib::placeholders::_1;
 using websocketpp::lib::placeholders::_2;
@@ -182,6 +185,10 @@ my $tlsPort = $model->getParameterByName("tlsPort");
 # Default: 443
 $tlsPort = $tlsPort ? $tlsPort->getValueAt(0)->getCppExpression() : 443;
 
+my $tlsCipherWhitelist = $model->getParameterByName("tlsCipherWhitelist");
+# Default: Empty string
+$tlsCipherWhitelist = $tlsCipherWhitelist ? $tlsCipherWhitelist->getValueAt(0)->getCppExpression() : "";
+
 my $certificateFileName = $model->getParameterByName("certificateFileName");
 # Default: Default is to read ws-server.pem from the etc sub-directory of the application.
 $certificateFileName = $certificateFileName ? $certificateFileName->getValueAt(0)->getCppExpression() : "";
@@ -343,6 +350,12 @@ MY_OPERATOR::MY_OPERATOR() {
 	// will lead to a compiler error. So, only for the string based copy from a
 	// perl variable, the following check must be done for an empty string and
 	// then it should be done as shown below.
+	<% if ($tlsCipherWhitelist eq "") { %>	
+	tlsCipherWhitelist = "";
+	<% } else { %>
+	tlsCipherWhitelist = <%=$tlsCipherWhitelist%>;
+	<%}%>
+
 	<% if ($certificateFileName eq "") { %>	
 	certificateFileName = "";
 	<% } else { %>
@@ -416,6 +429,7 @@ MY_OPERATOR::MY_OPERATOR() {
 		"-->Channel " << boost::to_string(udpChannelNumber) <<
 		". Following are the user configured operator parameters: "
 		"tlsPort=" << tlsPort <<
+		", tlsCipherWhitelist=" << tlsCipherWhitelist <<
 		", certificateFileName=" << certificateFileName <<
 		", certificatePassword length=" << certificatePassword.length() <<
 		", trustedClientCertificateFileName=" << trustedClientCertificateFileName <<
@@ -544,6 +558,15 @@ void MY_OPERATOR::prepareToShutdown() {
 	// This operator is being shutdown now.
 	// We can empty the following containers.
 	client_connections_map.clear();
+	
+	// At this time, we can stop the boost ASIO run loop that
+	// got started below in the ws_server method of this sink operator.
+	if(iosRunLoopStarted == true) {
+		// It will stop the ios run loop and make the
+		// sink operator ws_server thread exit. It should also help
+		// in the immediate closure of the TLS and non-TLS ports.
+		ios.stop();
+	}
 } // End of prepareToShutdown.
 
 // Processing for source and threaded operators   
@@ -851,9 +874,8 @@ void MY_OPERATOR::ws_server() {
 	// corresponding close brace at the end of the code block below.
 	//
 	// while(!getPE().getShutdownRequested()) {
-	// Set up an external io_service to run both endpoints on. This is not
-	// strictly necessary, but simplifies thread management a bit.
-	boost::asio::io_service ios;
+	
+	iosRunLoopStarted = false;
 
 	// If the user opted for an additional non-TLS (plain) 
 	// Websocket endpoint, let us create that as well.
@@ -972,6 +994,7 @@ void MY_OPERATOR::ws_server() {
 	// This will block until the server socket gets closed.
 	// For additional details, please refer to the commentary and 
 	// logic in the prepareToShutdown method
+	iosRunLoopStarted = true;
 	ios.run();
 	// }
 } // End of ws_server
@@ -1261,6 +1284,27 @@ MY_OPERATOR::context_ptr MY_OPERATOR::on_tls_init(
 						| boost::asio::ssl::context::no_tlsv1_1
 						| boost::asio::ssl::context::single_dh_use);
 		ctx->set_password_callback(bind(&MY_OPERATOR::get_private_key_password, this));
+		
+		// If the user has configured specific TLS ciphers to be used when
+		// establishing the TLS connection, then we can set it now.
+		// Reference URLs:
+		// https://stackoverflow.com/questions/50058521/using-specific-cipher-for-ssl-tls-in-boostasio
+		// https://www.ibm.com/support/knowledgecenter/SSB23S_1.1.0.2020/gtpc2/cpp_ssl_ctx_set_cipher_list.html
+		// https://www.ibm.com/support/knowledgecenter/SSB23S_1.1.0.2020/gtps7/s5sple1.html
+		if(tlsCipherWhitelist.length() > 0) {
+			// e-g: "DES-CBC3-SHA,AES128-SHA,AES256-SHA"
+			if (SSL_CTX_set_cipher_list(ctx->native_handle(), 
+				tlsCipherWhitelist.c_str()) <= 0) {
+				std::cout << "Operator " << operatorPhysicalName <<
+					"-->Channel " << boost::to_string(udpChannelNumber) << 
+					"-->" << "Error in setting user provided TLS cipher list." << std::endl;
+				
+				throw std::runtime_error(
+					std::string("WebSocketSink_cpp.cgt: Error in on_tls_init: ") +
+					"Error in setting user provided TLS cipher list.");			
+			}
+		}
+
 		ctx->use_certificate_chain_file(certificateFileName.c_str());
 		ctx->use_private_key_file(certificateFileName.c_str(),
 			boost::asio::ssl::context::pem);

com.ibm.streamsx.websocket/com.ibm.streamsx.websocket.op/WebSocketSink/WebSocketSink_h.cgt

@@ -8,7 +8,7 @@
 /*
 ============================================================
 First created on: Apr/27/2020
-Last modified on: Aug/31/2020
+Last modified on: Sep/21/2020
 
 This include file contains the necessary inclusion of 
 other C++ header files, declaration of member variables 
@@ -57,6 +57,11 @@ public:
 	std::string operatorPhysicalName;
 	SPL::int32 udpChannelNumber;
 	SPL::uint32 tlsPort;
+	// A comma separated approved TLS/SSL ciphers can be specified.
+	// If there is a need to avoid using insecure vulnerable ciphers,
+	// this can be used to specify the approved ciphers to be used.
+	// e-g: "DES-CBC3-SHA,AES128-SHA,AES256-SHA"
+	std::string tlsCipherWhitelist;
 	std::string certificateFileName;
 	std::string certificatePassword;
 	std::string trustedClientCertificateFileName;
@@ -79,6 +84,10 @@ public:
 	SPL::uint32 maxClientConnectionsAllowed;
 	server_non_tls endpoint_non_tls;
 	server_tls endpoint_tls;
+	// Set up an external io_service to run the endpoint on. This is not
+	// strictly necessary, but simplifies thread management a bit.
+	boost::asio::io_service ios;
+	bool iosRunLoopStarted;
 	SPL::boolean tlsEndpointStarted;
 	SPL::boolean nonTlsEndpointStarted;
 	SPL::int64 timeOfPreviousStaleConnectionRemoval;

com.ibm.streamsx.websocket/com.ibm.streamsx.websocket.op/WebSocketSource/WebSocketSource.xml

@@ -484,6 +484,16 @@
         <type>boolean</type>
         <cardinality>1</cardinality>
       </parameter>
+      
+      <parameter>
+        <name>tlsCipherWhitelist</name>
+        <description>This parameter can be used to specify a string containing one or more comma separated TLS/SSL ciphers that should be used during TLS/SSL connection negotiations with clients. It is handy when there is a need to avoid using ciphers that are found to have security vulnerabilities. (Default is an empty string.)</description>
+        <optional>true</optional>
+        <rewriteAllowed>true</rewriteAllowed>
+        <expressionMode>AttributeFree</expressionMode>
+        <type>rstring</type>
+        <cardinality>1</cardinality>
+      </parameter>
     </parameters>
 
     <inputPorts>

com.ibm.streamsx.websocket/com.ibm.streamsx.websocket.op/WebSocketSource/WebSocketSource_cpp.cgt

@@ -8,7 +8,7 @@
 /*
  ============================================================
  First created on: Feb/22/2020
- Last modified on: Aug/31/2020
+ Last modified on: Sep/22/2020
 
  This particular operator (WebSocketSource) is used to
  receive either text data or binary data from one or more
@@ -52,6 +52,9 @@
 #include <boost/exception/to_string.hpp>
 #include <boost/thread/thread.hpp>
 
+// Necessary headers for setting user specified TLS ciphers.
+#include <openssl/ssl.h>
+
 // For the core logic below, we will need these namespaces from websocketpp.
 using websocketpp::lib::placeholders::_1;
 using websocketpp::lib::placeholders::_2;
@@ -508,6 +511,10 @@ my $tlsPort = $model->getParameterByName("tlsPort");
 # Default: 443
 $tlsPort = $tlsPort ? $tlsPort->getValueAt(0)->getCppExpression() : 443;
 
+my $tlsCipherWhitelist = $model->getParameterByName("tlsCipherWhitelist");
+# Default: Empty string
+$tlsCipherWhitelist = $tlsCipherWhitelist ? $tlsCipherWhitelist->getValueAt(0)->getCppExpression() : "";
+
 my $certificateFileName = $model->getParameterByName("certificateFileName");
 # Default: Default is to read ws-server.pem from the etc sub-directory of the application.
 $certificateFileName = $certificateFileName ? $certificateFileName->getValueAt(0)->getCppExpression() : "";
@@ -735,6 +742,12 @@ MY_OPERATOR::MY_OPERATOR() {
 	// will lead to a compiler error. So, only for the string based copy from a
 	// perl variable, the following check must be done for an empty string and
 	// then it should be done as shown below.
+	<% if ($tlsCipherWhitelist eq "") { %>	
+	tlsCipherWhitelist = "";
+	<% } else { %>
+	tlsCipherWhitelist = <%=$tlsCipherWhitelist%>;
+	<%}%>
+	
 	<% if ($certificateFileName eq "") { %>	
 	certificateFileName = "";
 	<% } else { %>
@@ -821,6 +834,7 @@ MY_OPERATOR::MY_OPERATOR() {
 		"-->Channel " << boost::to_string(udpChannelNumber) <<
 		". Following are the user configured operator parameters: "
 		"tlsPort=" << tlsPort <<
+		", tlsCipherWhitelist=" << tlsCipherWhitelist <<
 		", certificateFileName=" << certificateFileName <<
 		", certificatePassword length=" << certificatePassword.length() <<
 		", trustedClientCertificateFileName=" << trustedClientCertificateFileName <<
@@ -961,6 +975,15 @@ void MY_OPERATOR::prepareToShutdown() {
 	// This operator is being shutdown now.
 	// We can empty the following containers.
 	client_connections_map.clear();
+	
+	// At this time, we can stop the boost ASIO run loop that
+	// got started below in the process method of this source operator.
+	if(iosRunLoopStarted == true) {
+		// It will stop the ios run loop and make the
+		// source operator thread exit. It should also help
+		// in the immediate closure of the TLS and non-TLS ports.
+		ios.stop();
+	}
 } // End of prepareToShutdown.
 
 // Processing for source and threaded operators   
@@ -995,10 +1018,9 @@ void MY_OPERATOR::process(uint32_t idx) {
 	// corresponding close brace at the end of the code block below.
 	//
 	// while(!getPE().getShutdownRequested()) {
-	// Set up an external io_service to run both endpoints on. This is not
-	// strictly necessary, but simplifies thread management a bit.
-	boost::asio::io_service ios;
 
+	iosRunLoopStarted = false;
+	
 	// If the user opted for an additional non-TLS (plain) 
 	// Websocket endpoint, let us create that as well.
 	// This non_tls insecure endpoint is here for a very 
@@ -1128,6 +1150,7 @@ void MY_OPERATOR::process(uint32_t idx) {
 	// This will block until the server socket gets closed.
 	// For additional details, please refer to the commentary and 
 	// logic in the prepareToShutdown method
+	iosRunLoopStarted = true;
 	ios.run();
 	// }
 } // End of process (for source operators)
@@ -2297,6 +2320,27 @@ MY_OPERATOR::context_ptr MY_OPERATOR::on_tls_init(
 						| boost::asio::ssl::context::no_tlsv1_1
 						| boost::asio::ssl::context::single_dh_use);
 		ctx->set_password_callback(bind(&MY_OPERATOR::get_private_key_password, this));
+
+		// If the user has configured specific TLS ciphers to be used when
+		// establishing the TLS connection, then we can set it now.
+		// Reference URLs:
+		// https://stackoverflow.com/questions/50058521/using-specific-cipher-for-ssl-tls-in-boostasio
+		// https://www.ibm.com/support/knowledgecenter/SSB23S_1.1.0.2020/gtpc2/cpp_ssl_ctx_set_cipher_list.html
+		// https://www.ibm.com/support/knowledgecenter/SSB23S_1.1.0.2020/gtps7/s5sple1.html
+		if(tlsCipherWhitelist.length() > 0) {
+			// e-g: "DES-CBC3-SHA,AES128-SHA,AES256-SHA"
+			if (SSL_CTX_set_cipher_list(ctx->native_handle(), 
+				tlsCipherWhitelist.c_str()) <= 0) {
+				std::cout << "Operator " << operatorPhysicalName <<
+					"-->Channel " << boost::to_string(udpChannelNumber) << 
+					"-->" << "Error in setting user provided TLS cipher list." << std::endl;
+				
+				throw std::runtime_error(
+					std::string("WebSocketSource_cpp.cgt: Error in on_tls_init: ") +
+					"Error in setting user provided TLS cipher list.");			
+			}
+		}
+		
 		// Let us now load the server's private key and 
 		// public certificate holding PEM file.
 		ctx->use_certificate_chain_file(certificateFileName.c_str());