updates

Author: bingzhang

CHANGELOG.md

@@ -6,6 +6,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 
 ## Unreleased
 ### Added
+- add script to test Clowder extraction.
 - add script to submit files to Clowder geo extraction.
 - Added ability to delete extractor, both from api and GUI.
   [CATS-1044](https://opensource.ncsa.illinois.edu/jira/browse/CATS-1044)

scripts/tester/tester.sh

@@ -28,69 +28,71 @@ post_message() {
 
 # ------------------------ Create dataset ------------------------
 DATASET_ID=$(curl -X POST -H "Content-Type: application/json" \
-	-d '{"name":"Temporary Test Dataset", "description":"Created automatically by test script."}' \
-	$CLOWDER_URL/api/datasets/createempty?key=$CLOWDER_KEY)
+    -d '{"name":"Temporary Test Dataset", "description":"Created automatically by test script."}' \
+    $CLOWDER_URL/api/datasets/createempty?key=$CLOWDER_KEY)
 DATASET_ID=$(echo $DATASET_ID  | jq '.id' | sed s/\"//g)
 echo "Dataset ID: $DATASET_ID"
 
 # ------------------------ Upload file ------------------------
 FILE_ID=$(curl -X POST -H "Content-Type: application/json" \
-	-F File=@$TARGET_FILE \
-	$CLOWDER_URL/api/uploadToDataset/$DATASET_ID?key=$CLOWDER_KEY&extract=0)
+    -F File=@$TARGET_FILE \
+    $CLOWDER_URL/api/uploadToDataset/$DATASET_ID?key=$CLOWDER_KEY&extract=0)
 FILE_ID=$(echo $FILE_ID | jq '.id' | sed s/\"//g)
 echo "File ID: $FILE_ID"
 
 # Validate upload
 FILE_UPLOADED=0
 RETRIES=0
 while [ $FILE_UPLOADED = 0 ]; do
-	RESPONSE=$(curl -X GET -H "Content-Type: application/json" \
-		$CLOWDER_URL/api/files/$FILE_ID/metadata?key=$CLOWDER_KEY)
-	RESPONSE=$(echo $RESPONSE | jq '.status' | sed s/\"//g)
-	if [ "$RESPONSE" = "PROCESSED" ]; then
-		FILE_UPLOADED=1
-	fi
-	RETRIES=$((RETRIES+1))
-	if [ $RETRIES = 12 ]; then
-	  echo "File upload not PROCESSED after 2 minutes. There may be a problem. Deleting dataset."
-	  curl -X DELETE $CLOWDER_URL/api/datasets/$DATASET_ID?key=$CLOWDER_KEY
-	  post_message("Upload+extract test script failing on $CLOWDER_URL/files/$FILE_ID (status is not PROCESSED)")
-	  exit 1
-	fi
-	echo "File upload not complete; checking again in 10 seconds."
-	sleep 10
+    RESPONSE=$(curl -X GET -H "Content-Type: application/json" \
+        $CLOWDER_URL/api/files/$FILE_ID/metadata?key=$CLOWDER_KEY)
+    RESPONSE=$(echo $RESPONSE | jq '.status' | sed s/\"//g)
+    if [ "$RESPONSE" = "PROCESSED" ]; then
+        FILE_UPLOADED=1
+    fi
+    RETRIES=$((RETRIES+1))
+    if [ $RETRIES = 12 ]; then
+      echo "File upload not PROCESSED after 2 minutes. There may be a problem. Deleting dataset."
+      curl -X DELETE $CLOWDER_URL/api/datasets/$DATASET_ID?key=$CLOWDER_KEY
+      post_message "Upload+extract test script failing on $CLOWDER_URL\/files\/$FILE_ID (status is not PROCESSED)"
+      exit 1
+    fi
+    echo "File upload not complete; checking again in 10 seconds."
+    sleep 10
 done
 echo "File upload complete."
 
 # ------------------------ Submit for extraction ------------------------
 curl -X POST -H "Content-Type: application/json" \
-	-d '{"extractor": "ncsa.file.digest"}' \
-	$CLOWDER_URL/api/files/$FILE_ID/extractions?key=$CLOWDER_KEY
+    -d '{"extractor": "ncsa.file.digest"}' \
+    $CLOWDER_URL/api/files/$FILE_ID/extractions?key=$CLOWDER_KEY
 
 # Validate extraction
 FILE_EXTRACTED=0
 RETRIES=0
-while [ FILE_EXTRACTED = 0 ]; do
-	RESPONSE=$(curl -X GET -H "Content-Type: application/json" \
-		$CLOWDER_URL/api/extractions/$FILE_ID/status?key=$CLOWDER_KEY)
-	RESPONSE=$(echo $RESPONSE | jq '.ncsa.file.digest' | sed s/\"//g)
-	if [ "$RESPONSE" = "DONE" ]; then
-		FILE_EXTRACTED=1
-	fi
-	RETRIES=$((RETRIES+1))
-	if [ $RETRIES = 24 ]; then
-	  echo "File extraction not DONE after 4 minutes. There may be a problem. Deleting dataset."
-	  curl -X DELETE $CLOWDER_URL/api/datasets/$DATASET_ID?key=$CLOWDER_KEY
-	  post_message("Upload+extract test script failing on $CLOWDER_URL/files/$FILE_ID (extractor not DONE)")
-	  exit 1
-	fi
-	echo "File extraction not complete; checking again in 10 seconds."
-	sleep 10
+while [ $FILE_EXTRACTED -eq 0 ]; do
+    RESPONSE=$(curl -X GET -H "Content-Type: application/json" \
+        $CLOWDER_URL/api/extractions/$FILE_ID/status?key=$CLOWDER_KEY)
+    echo $RESPONSE
+    RESPONSE=$(echo $RESPONSE | jq '."ncsa.file.digest"' | sed s/\"//g)
+    if [ "$RESPONSE" = "DONE" ]; then
+        FILE_EXTRACTED=1
+        post_message "Extractor: [ncsa.file.digest] success $CLOWDER_URL/files/$FILE_ID"
+    fi
+    RETRIES=$((RETRIES+1))
+    if [ $RETRIES = 24 ]; then
+      echo "File extraction not DONE after 4 minutes. There may be a problem. Deleting dataset."
+      curl -X DELETE $CLOWDER_URL/api/datasets/$DATASET_ID?key=$CLOWDER_KEY
+      post_message "Upload+extract test script failing on $CLOWDER_URL/files/$FILE_ID (extractor not DONE)"
+      exit 1
+    fi
+    echo "File extraction not complete; checking again in 10 seconds."
+    sleep 10
 done
 echo "File extraction complete."
-	
+    
 
 # ------------------------ Delete dataset ------------------------
 curl -X DELETE $CLOWDER_URL/api/datasets/$DATASET_ID?key=$CLOWDER_KEY
 
-echo "Test complete."
+echo "Test complete."