Changes done for DPS to be used in Streams 4.x

nysenthil · nysenthil · commit aa22b2332b63 · 2015-04-16T00:30:19.000-04:00
diff --git a/com.ibm.streamsx.dps/.toolkitList b/com.ibm.streamsx.dps/.toolkitList
@@ -1,6 +1,3 @@
-com.ibm.streamsx.dps/1.0.9
-spl/1.1.0
-058_data_sharing_between_non_fused_spl_custom_and_cpp_primitive_operators/1.0.0
-061_data_sharing_between_non_fused_spl_custom_operators_and_a_native_function/1.0.0
-062_data_sharing_between_non_fused_spl_custom_and_java_primitive_operators/1.0.0
-dps_test_1/1.0.0
+com.ibm.streamsx.dps/2.0.9
+spl/1.2.0
+dps_test_1/1.0.1
diff --git a/com.ibm.streamsx.dps/doc/dps-usage-tips.txt b/com.ibm.streamsx.dps/doc/dps-usage-tips.txt
@@ -22,7 +22,9 @@ Following are the NoSQL K/V stores supported by the DPS toolkit:
  
 A companion IBM developerWorks article about this toolkit: http://tinyurl.com/nxrf3gg
 
-[Apr/12/2015]                  [DPS toolkit version 1.0.9]                  [Share and prosper]
+This version of the DPS toolkit works with Streams 4.x and higher versions.
+
+[Apr/15/2015]                  [DPS toolkit version 2.0.9]                  [Share and prosper]
 ================================================================================================
 This usage tips file provides high-level implementation details about the Streams DPS toolkit.
 Main effort here is to provide a Streams toolkit that will allow data sharing
@@ -132,7 +134,7 @@ DPS toolkit is routinely compiled and tested in our Streams lab using the follow
 a) RedHat Enterprise Linux 6.4 (or an equivalent CentOS version) + gcc version 4.4.7 20120313 (Red Hat 4.4.7-3) (GCC)
 b) RedHat Enterprise Linux 6.4 for IBM Power machines + gcc version 4.4.7 20120313 (Red Hat 4.4.7-3) (GCC)
    [Dec/23/2014: DPS toolkit version 1.0.6 and later can't be used with RHEL5 and CentOS5.]
-c) IBM InfoSphere Streams Version 3.x
+c) IBM InfoSphere Streams Version 4.x and higher versions
 d) memcached server version 1.4.15 with libmemcached client version 1.0.16
 e) redis server version 2.8.19 with hiredis C client version 0.12.1
 f) Cassandra server version 2.1.4 with C++ client driver 1.0.1 (April/2015) from DataStax
@@ -452,7 +454,7 @@ If you installed Cassandra, you can start Cassandra as shown below.
        1.7u25, from here and unzip it in your own directory and set the JAVA_HOME environment variable to that new Java directory.
        
        http://www.oracle.com/technetwork/java/javase/downloads/jdk7-downloads-1880260.html
-       (As of April/2015, the latest version of Java SDK from this URL is 1.7.0_75)
+       (As of April/2015, the latest version of Java SDK from this URL is 1.7.0_79)
        
     d) You can now start Cassandra to run in the background using this command: $CASSANDRA_HOME/bin/cassandra
     
@@ -970,9 +972,16 @@ you can skip steps 1 and 2 below.
    This will create a new dps-helper.jar file in the impl/java/bin directory. This is needed to access the DPS
    functions from a Java primitive operator.
 
-4) Switch to <your_base_dir>/com.ibm.streamsx.dps/samples/dps_test1/bin directory and run this script: ./build-distributed.sh.
-5) You have to edit the <your_base_dir>/com.ibm.streamsx.dps/samples/dps_test1/etc/no-sql-kv-store-servers.cfg file and add the names or the
-   IP addresses of your memcached or redis or Cassandra or Cloudant or HBase or Mongo servers. Then, save the file.
+4) You have to edit the <your_base_dir>/com.ibm.streamsx.dps/samples/dps_test1/etc/no-sql-kv-store-servers.cfg file and add the names or the
+   IP addresses of your memcached or redis or Cassandra or Cloudant or HBase or Mongo or Couchbase or Aerospike or Redis-Cluster servers.
+   Then, save the file.
+5) Switch to <your_base_dir>/com.ibm.streamsx.dps/samples/dps_test1/bin directory and run this script: ./build-distributed.sh.
+   [Note: Whenever you change the DPS configuration file as shown in step 4, you must build the application and ensure that
+    the application bundle (sab) file in the output directory has a new file modification time. If that file's modification time
+    indicates that the sab file never got updated, then you must completely remove 
+    <your_base_dir>/com.ibm.streamsx.dps/samples/dps_test1/output directory and build again. If you don't do this, your DPS configuration
+    changes will not take effect. As of Apr/2015, this is a new requirement/restriction in Streams 4.x to update the
+    application bundle with your modified DPS configuration file. This requirement/restriction will be eliminated in a future release.]
 6) To test the application, you can run this script from <your_base_dir>/com.ibm.streamsx.dps/samples/dps_test1/bin: ./run-distributed.sh -i <YOUR_STREAMS_INSTANCE_NAME>
 7) You can verify the application results in the Streams PE log files.
    --> You have to first read the SPLMM file and understand which operator writes what in the PE log files.
diff --git a/com.ibm.streamsx.dps/impl/java/bin/dps-helper.jar b/com.ibm.streamsx.dps/impl/java/bin/dps-helper.jar
diff --git a/com.ibm.streamsx.dps/impl/lib/ppc64.RHEL6/libDistributedProcessStoreLib.so b/com.ibm.streamsx.dps/impl/lib/ppc64.RHEL6/libDistributedProcessStoreLib.so
diff --git a/com.ibm.streamsx.dps/impl/lib/ppc64.RHEL6/libDpsJavaLibLoader.so b/com.ibm.streamsx.dps/impl/lib/ppc64.RHEL6/libDpsJavaLibLoader.so
diff --git a/com.ibm.streamsx.dps/impl/lib/x86_64.RHEL6/libDistributedProcessStoreLib.so b/com.ibm.streamsx.dps/impl/lib/x86_64.RHEL6/libDistributedProcessStoreLib.so
diff --git a/com.ibm.streamsx.dps/impl/lib/x86_64.RHEL6/libDpsJavaLibLoader.so b/com.ibm.streamsx.dps/impl/lib/x86_64.RHEL6/libDpsJavaLibLoader.so
diff --git a/com.ibm.streamsx.dps/impl/src/DistributedProcessStore.cpp b/com.ibm.streamsx.dps/impl/src/DistributedProcessStore.cpp
@@ -74,9 +74,19 @@ namespace distributed
     conf /= "../etc/no-sql-kv-store-servers.cfg";
     std::string confFile = conf.string();
     */
-    // Refer to the etc directory with a relative path from the default current directory (i.e. data directory)
-	std::string confFile = "../etc/no-sql-kv-store-servers.cfg";
- 
+	std::string appDirectory = ProcessingElement::pe().getApplicationDirectory();
+	/*
+	std::string dataDirectory = ProcessingElement::pe().getDataDirectory();
+	std::string outputDirectory = ProcessingElement::pe().getOutputDirectory();
+	std::string toolkitDirectory = ProcessingElement::pe().getToolkitDirectory();
+	cout << "App directory=" << appDirectory << endl;
+	cout << "Data directory=" << dataDirectory << endl;
+	cout << "Output directory=" << outputDirectory << endl;
+	cout << "Toolkit directory=" << toolkitDirectory << endl;
+	*/
+	// Refer to the etc directory with a relative path from the application directory.
+	std::string confFile = appDirectory + "/etc/no-sql-kv-store-servers.cfg";
+
     // Format of this file is as shown below.
     // Several comment lines beginning with a # character.
     // Then, the very first non-comment line will tell us the
diff --git a/com.ibm.streamsx.dps/info.xml b/com.ibm.streamsx.dps/info.xml
@@ -3,8 +3,8 @@
   <info:identity>
     <info:name>com.ibm.streamsx.dps</info:name>
     <info:description></info:description>
-    <info:version>1.0.9</info:version>
-    <info:requiredProductVersion>2.0.0</info:requiredProductVersion>
+    <info:version>2.0.9</info:version>
+    <info:requiredProductVersion>4.0.0.0</info:requiredProductVersion>
   </info:identity>
   <info:dependencies/>
 </info:toolkitInfoModel>
diff --git a/com.ibm.streamsx.dps/samples/dps_test_1/.toolkitList b/com.ibm.streamsx.dps/samples/dps_test_1/.toolkitList
@@ -1,6 +1,3 @@
-dps_test_1/1.0.0
-spl/1.1.0
-058_data_sharing_between_non_fused_spl_custom_and_cpp_primitive_operators/1.0.0
-061_data_sharing_between_non_fused_spl_custom_operators_and_a_native_function/1.0.0
-062_data_sharing_between_non_fused_spl_custom_and_java_primitive_operators/1.0.0
-com.ibm.streamsx.dps/1.0.9
+dps_test_1/1.0.1
+spl/1.2.0
+com.ibm.streamsx.dps/2.0.9
diff --git a/com.ibm.streamsx.dps/samples/dps_test_1/bin/build-distributed.sh b/com.ibm.streamsx.dps/samples/dps_test_1/bin/build-distributed.sh
@@ -1,3 +1,3 @@
 cd ../
 rm -rf ./output
-sc -a -M DpsTest1 --output-directory=./output/DpsTest1/Distributed -t ../../
+sc -a -M DpsTest1 --data-directory=./data --output-directory=./output/DpsTest1/Distributed -t ../../
diff --git a/com.ibm.streamsx.dps/samples/dps_test_1/bin/run-distributed.sh b/com.ibm.streamsx.dps/samples/dps_test_1/bin/run-distributed.sh
@@ -1,15 +1,20 @@
 s=0	# Number of options to shift out
-while getopts "i:p:o:r:f:s:n:h" options; do
+while getopts "d:i:h" options; do
     case $options in
+    d) streams_domain_name=$OPTARG
+       let s=s+2
+       ;;
     i) streams_instance_name=$OPTARG
        let s=s+2
        ;;
 
     h | * ) echo "
 Command line arguments
-  -i STRING     streamsInstanceName                       		(silver_stream)
+  -d STRING     streamsDomainName     (big_space)
+  -i STRING     streamsInstanceName   (silver_stream)
  
   e-g:
+  -d big_space
   -i silver_stream
 "
         exit 1
@@ -18,6 +23,16 @@ Command line arguments
 done
 shift $s
 
+# Validate the streams domain name  entered by the user.
+if [ "$streams_domain_name" == "" ];
+then
+   echo "Missing or wrong streams domain name via the -d option."
+   echo "Your streams domain name must be specified."
+   echo ""
+   echo "Get help using -h option."
+   exit 1
+fi
+
 # Validate the streams instance name entered by the user.
 if [ "$streams_instance_name" == "" ];
 then
@@ -35,8 +50,8 @@ echo "Ensuring that the Streams instance '$streams_instance_name' is running ...
 # If Streams instance is already started and running, it will display a warning on
 # the stderr console. Let us suppress that warning by redirecting stderr and 
 # stdout to a null device.
-streamtool startinstance -i $streams_instance_name &> /dev/null
+streamtool startinstance -d $streams_domain_name -i $streams_instance_name &> /dev/null
 
-streamtool submitjob -i $streams_instance_name ../output/DpsTest1/Distributed/DpsTest1.adl
+streamtool submitjob -d $streams_domain_name -i $streams_instance_name ../output/DpsTest1/Distributed/DpsTest1.sab 
 
 echo "****** You can check the results from this run inside the PE stdouterr log files in the Streams application log directory.  ******"
diff --git a/com.ibm.streamsx.dps/samples/dps_test_1/bin/stop-streams-app.sh b/com.ibm.streamsx.dps/samples/dps_test_1/bin/stop-streams-app.sh
@@ -3,16 +3,21 @@
 # Use this script to stop all the streams jobs that are running.
 # ========================
 s=0	# Number of options to shift out
-while getopts "i:h" options; do
+while getopts "d:i:h" options; do
     case $options in
+    d) streams_domain_name=$OPTARG
+       let s=s+2
+       ;;
     i) streams_instance_name=$OPTARG
        let s=s+2
        ;;
     h | * ) echo "
 Command line arguments
-  -i STRING     streamsInstanceName (silver_stream)
+  -d STRING       streamsDomainName   (big_space)
+  -i STRING     streamsInstanceName   (silver_stream)
 
   e-g:
+  -d big_space
   -i silver_stream
 "
         exit 1
@@ -21,6 +26,16 @@ Command line arguments
 done
 shift $s
 
+# Validate the streams domain name  entered by the user.
+if [ "$streams_domain_name" == "" ];
+then
+   echo "Missing or wrong streams domain name via the -d option."
+   echo "Your streams domain name must be specified."
+   echo ""
+   echo "Get help using -h option."
+   exit 1
+fi
+
 # Validate the streams instance name  entered by the user.
 if [ "$streams_instance_name" == "" ];
 then
@@ -35,9 +50,9 @@ fi
 # Let us collect information about all the jobs running under this streams instance.
 temp_my_streams_jobs_info_file="$HOME/temp-my-streams-jobs-info.txt"
 rm -f "$temp_my_streams_jobs_info_file"
-streamtool lsjobs -i $streams_instance_name | grep -i Running > "$temp_my_streams_jobs_info_file"
+streamtool lsjobs -d $streams_domain_name -i $streams_instance_name | grep -i Running > "$temp_my_streams_jobs_info_file"
 # Get the total number of jobs running under this instance.
-total_jobs_cnt=$(streamtool lsjobs -i $streams_instance_name | grep -i Running | wc -l)
+total_jobs_cnt=$(streamtool lsjobs -d $streams_domain_name -i $streams_instance_name | grep -i Running | wc -l)
 
 # Read the jobs info file and get the job ids running under the given streams instance.
 job_id=99999
@@ -48,7 +63,7 @@ do
    job_id=$(echo $line | awk '{print $1}')
    echo "Stopping job id "${job_id}
    # Cancel the Streams job.
-   streamtool canceljob -i $streams_instance_name $job_id
+   streamtool canceljob -d $streams_domain_name -i $streams_instance_name $job_id
    let job_stopped_cnt=job_stopped_cnt+1
    echo "Stopped job $job_stopped_cnt of $total_jobs_cnt."
    
diff --git a/com.ibm.streamsx.dps/samples/dps_test_1/bin/stop-streams-instance.sh b/com.ibm.streamsx.dps/samples/dps_test_1/bin/stop-streams-instance.sh
@@ -3,16 +3,21 @@
 # Use this script to stop your streams instance.
 # ========================
 s=0	# Number of options to shift out
-while getopts "i:h" options; do
+while getopts "d:i:h" options; do
     case $options in
+    d) streams_domain_name=$OPTARG
+       let s=s+2
+       ;;
     i) streams_instance_name=$OPTARG
        let s=s+2
        ;;
     h | * ) echo "
 Command line arguments
-  -i STRING     streamsInstanceName (silver_stream)
+  -d STRING     streamsDomainName     (big_space)
+  -i STRING     streamsInstanceName   (silver_stream)
 
   e-g:
+  -d big_space
   -i silver_stream
 "
         exit 1
@@ -21,6 +26,16 @@ Command line arguments
 done
 shift $s
 
+# Validate the streams domain name  entered by the user.
+if [ "$streams_domain_name" == "" ];
+then
+   echo "Missing or wrong streams domain name via the -d option."
+   echo "Your streams domain name must be specified."
+   echo ""
+   echo "Get help using -h option."
+   exit 1
+fi
+
 # Validate the streams instance name  entered by the user.
 if [ "$streams_instance_name" == "" ];
 then
@@ -32,6 +47,6 @@ then
 fi
 
 # Let us stop the instance now.
-streamtool stopinstance -i $streams_instance_name
+streamtool stopinstance -d $streams_domain_name -i $streams_instance_name
 
 date
diff --git a/com.ibm.streamsx.dps/samples/dps_test_1/info.xml b/com.ibm.streamsx.dps/samples/dps_test_1/info.xml
@@ -3,13 +3,13 @@
   <info:identity>
     <info:name>dps_test_1</info:name>
     <info:description></info:description>
-    <info:version>1.0.0</info:version>
-    <info:requiredProductVersion>2.0.0</info:requiredProductVersion>
+    <info:version>1.0.1</info:version>
+    <info:requiredProductVersion>4.0.0.0</info:requiredProductVersion>
   </info:identity>
   <info:dependencies>
     <info:toolkit>
       <common:name>com.ibm.streamsx.dps</common:name>
-      <common:version>[1.0.0,2.0.0)</common:version>
+      <common:version>[2.0.0,3.0.0)</common:version>
     </info:toolkit>
   </info:dependencies>
 </info:toolkitInfoModel>
diff --git a/com.ibm.streamsx.dps/samples/dps_test_1/toolkit.xml b/com.ibm.streamsx.dps/samples/dps_test_1/toolkit.xml
@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no" ?>
-<toolkitModel xmlns="http://www.ibm.com/xmlns/prod/streams/spl/toolkit" productVersion="3.2.1.0" xmlns:common="http://www.ibm.com/xmlns/prod/streams/spl/common" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+<toolkitModel xmlns="http://www.ibm.com/xmlns/prod/streams/spl/toolkit" productVersion="4.0.0.0" xmlns:common="http://www.ibm.com/xmlns/prod/streams/spl/common" xmlns:ti="http://www.ibm.com/xmlns/prod/streams/spl/toolkitInfo" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
 
-  <toolkit name="dps_test_1" requiredProductVersion="2.0.0" version="1.0.0">
+  <toolkit name="dps_test_1" requiredProductVersion="4.0.0.0" version="1.0.1">
     <description/>
     <uriTable>
       <uri index="1" value="DpsTest1.spl"/>
@@ -33,8 +33,20 @@
     </namespace>
     <dependency>
       <common:name>com.ibm.streamsx.dps</common:name>
-      <common:version>[1.0.0,2.0.0)</common:version>
+      <common:version>[2.0.0,3.0.0)</common:version>
     </dependency>
+    <sabFiles>
+      <ti:include path="toolkit.xml" root="toolkitDir"/>
+      <ti:include path="impl/java/lib/**" root="toolkitDir"/>
+      <ti:include path="impl/java/bin/**" root="toolkitDir"/>
+      <ti:include path="impl/bin/**" root="toolkitDir"/>
+      <ti:include path="impl/lib/**" root="toolkitDir"/>
+      <ti:include path="impl/nl/*.dat" root="toolkitDir"/>
+      <ti:include path="etc/**" root="toolkitDir"/>
+      <ti:include path="lib/**" root="toolkitDir"/>
+      <ti:include path="nl/**" root="toolkitDir"/>
+      <ti:include path="opt/**" root="toolkitDir"/>
+    </sabFiles>
   </toolkit>
 
 </toolkitModel>
diff --git a/com.ibm.streamsx.dps/toolkit.xml b/com.ibm.streamsx.dps/toolkit.xml
@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no" ?>
-<toolkitModel xmlns="http://www.ibm.com/xmlns/prod/streams/spl/toolkit" productVersion="3.2.1.0" xmlns:common="http://www.ibm.com/xmlns/prod/streams/spl/common" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+<toolkitModel xmlns="http://www.ibm.com/xmlns/prod/streams/spl/toolkit" productVersion="4.0.0.0" xmlns:common="http://www.ibm.com/xmlns/prod/streams/spl/common" xmlns:ti="http://www.ibm.com/xmlns/prod/streams/spl/toolkitInfo" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
 
-  <toolkit name="com.ibm.streamsx.dps" requiredProductVersion="2.0.0" version="1.0.9">
+  <toolkit name="com.ibm.streamsx.dps" requiredProductVersion="4.0.0.0" version="2.0.9">
     <description/>
     <uriTable>
       <uri index="1" value="com.ibm.streamsx/lock/distributed/native.function/function.xml"/>
@@ -324,6 +324,18 @@
       </function>
     </namespace>
     <namespace name="com.ibm.streamsx"/>
+    <sabFiles>
+      <ti:include path="toolkit.xml" root="toolkitDir"/>
+      <ti:include path="impl/java/lib/**" root="toolkitDir"/>
+      <ti:include path="impl/java/bin/**" root="toolkitDir"/>
+      <ti:include path="impl/bin/**" root="toolkitDir"/>
+      <ti:include path="impl/lib/**" root="toolkitDir"/>
+      <ti:include path="impl/nl/*.dat" root="toolkitDir"/>
+      <ti:include path="etc/**" root="toolkitDir"/>
+      <ti:include path="lib/**" root="toolkitDir"/>
+      <ti:include path="nl/**" root="toolkitDir"/>
+      <ti:include path="opt/**" root="toolkitDir"/>
+    </sabFiles>
   </toolkit>
 
 </toolkitModel>
