Merge pull request #116 from datastax/issue/CDM-22

Issue/cdm 22

Author: mieslep

SIT/.gitignore

@@ -1,3 +1,5 @@
+output/
+
 setup.out
 setup.err
 execute.out

SIT/Makefile

@@ -1,5 +1,5 @@
 
-all: setup test_smoke env_teardown
+all: setup test_smoke_cmd test_features_cmd env_teardown
 
 setup: build env_setup
 
@@ -13,6 +13,10 @@ test_smoke: reset test_smoke_cmd
 test_smoke_cmd:
 	./test.sh -p smoke
 
+test_features: reset test_features_cmd
+test_features_cmd:
+	./test.sh -p features
+
 env_setup:
 	chmod -R 777 ./*.sh
 	./environment.sh -m setup -j ../target/cassandra-data-migrator*.jar

SIT/cdm-assert.sh

@@ -0,0 +1,101 @@
+#!/bin/bash
+
+assertCmd="egrep 'JobSession.* Final ' \${OUTPUT_FILE} | sed 's/^.*Final //'"
+
+_usage() {
+  cat <<EOF
+
+usage: $0 -f output_file -a assertFile [-d directory]
+
+Required
+  -f output_file   : a file with list of scenarios, same format as cdm.sh
+  -a assertFile    : a file with the assertions
+
+Optional
+  -d directory     : directory in which output_file and assertFile may be found
+
+==================
+assertFile Format
+==================
+Expected to contain the "Final" job session summary information, generated similar to
+    ${assertCmd}
+
+File might look like:
+---------------
+Read Record Count: 3
+Mismatch Record Count: 0
+Corrected Mismatch Record Count: 0
+Missing Record Count: 0
+Corrected Missing Record Count: 0
+Valid Record Count: 3
+Skipped Record Count: 0
+---------------
+
+EOF
+exit 1
+}
+
+while getopts "f:d:a:" opt; do
+  case $opt in 
+    f) OUTPUT_FILENAME="$OPTARG"
+       ;;
+    d) CONFIG_DIR="$OPTARG"
+       ;;
+    a) ASSERT_FILENAME="$OPTARG"
+       ;;
+    ?) echo "ERROR invalid option was specified - ${OPTARG}"
+       _usage
+       ;;
+  esac
+done
+
+argErrors=0
+if [[ -z "$OUTPUT_FILENAME" ]]; then
+  echo "missing -o output_file"
+  argErrors=1
+else
+  if [[ -z "${CONFIG_DIR}" ]]; then
+    OUTPUT_FILE="${OUTPUT_FILENAME}"
+  else
+    OUTPUT_FILE="${CONFIG_DIR}/${OUTPUT_FILENAME}"
+  fi
+  if [[ ! -r ${OUTPUT_FILE} ]]; then
+    echo "config file ${OUTPUT_FILE} not found, or is not readable"
+    argErrors=1
+  fi
+fi
+
+if [[ -z "$ASSERT_FILENAME" ]]; then
+  echo "missing -a assertFile"
+  argErrors=1
+else
+  if [[ -z "${CONFIG_DIR}" ]]; then
+    ASSERT_FILE="${ASSERT_FILENAME}"
+  else
+    ASSERT_FILE="${CONFIG_DIR}/${ASSERT_FILENAME}"
+  fi
+  if [[ ! -r ${ASSERT_FILE} ]]; then
+    echo "config file ${ASSERT_FILE} not found, or is not readable"
+    argErrors=1
+  fi
+fi
+
+
+if [ $argErrors -ne 0 ]; then
+  _usage
+fi
+
+WORKING_FILE="${ASSERT_FILENAME}".actual.out
+if [[ -n "${CONFIG_DIR}" ]]; then
+  WORKING_FILE="${CONFIG_DIR}/${WORKING_FILE}"
+fi
+
+eval ${assertCmd} > ${WORKING_FILE}
+
+diff -q ${ASSERT_FILE} ${WORKING_FILE} > /dev/null 2>&1
+rtn=$?
+if [ $rtn -eq 1 ]; then
+  echo "ERROR: CDM output $(basename ${OUTPUT_FILE}) differs from expected (expected vs actual):"
+  sdiff ${ASSERT_FILE} ${WORKING_FILE}
+  exit 1
+fi

SIT/cdm.sh

@@ -91,4 +91,4 @@ if [ $argErrors -ne 0 ]; then
   _usage
 fi
 
-spark-submit --properties-file "${PROPERTIES}" --master "local[*]" --class ${CLASS} /local/cassandra-data-migrator.jar
+spark-submit --properties-file "${PROPERTIES}" --master "local[*]" --class ${CLASS} /local/cassandra-data-migrator.jar

SIT/environment.sh

@@ -184,6 +184,7 @@ _createDockerCDM_Directory() {
   docker exec ${DOCKER_CDM} mkdir -p ${CDM_DIRECTORY}
   docker cp ${JAR} ${DOCKER_CDM}:${CDM_DIRECTORY}/${CDM_JARFILE}
   docker cp cdm.sh ${DOCKER_CDM}:${CDM_DIRECTORY}/cdm.sh
+  docker cp cdm-assert.sh ${DOCKER_CDM}:${CDM_DIRECTORY}/cdm-assert.sh
   if [[ "$(_testDockerCDM_Directory)" != "yes" ]]; then
     _fatal "Directory ${CDM_DIRECTORY} cannot be created, or ${JAR} cannot be copied to ${CDM_DIRECTORY}"
   fi  

SIT/features/01_constant_column/breakData.cql

@@ -0,0 +1,5 @@
+
+DELETE FROM target.feature_constant_column WHERE key='key2' AND const1='abcd';
+UPDATE target.feature_constant_column SET value='value999' WHERE key='key3' AND const1='abcd';
+SELECT * FROM target.feature_constant_column;
+

SIT/features/01_constant_column/cdm.fixData.assert

@@ -0,0 +1,7 @@
+Read Record Count: 3
+Mismatch Record Count: 1
+Corrected Mismatch Record Count: 1
+Missing Record Count: 1
+Corrected Missing Record Count: 1
+Valid Record Count: 1
+Skipped Record Count: 0

SIT/features/01_constant_column/cdm.migrateData.assert

@@ -0,0 +1,4 @@
+Read Record Count: 3
+Skipped Record Count: 0
+Write Record Count: 3
+Error Record Count: 0

SIT/features/01_constant_column/cdm.txt

@@ -0,0 +1,3 @@
+migrateData datastax.astra.migrate.Migrate migrate.properties
+validateData datastax.astra.migrate.DiffData migrate.properties
+fixData datastax.astra.migrate.DiffData fix.properties

SIT/features/01_constant_column/cdm.validateData.assert

@@ -0,0 +1,7 @@
+Read Record Count: 3
+Mismatch Record Count: 0
+Corrected Mismatch Record Count: 0
+Missing Record Count: 0
+Corrected Missing Record Count: 0
+Valid Record Count: 3
+Skipped Record Count: 0