SQS long polling, swallow CloudTrail validation messages

Swallow CloudTrail validation messages per suggestion by atward in issue #4
Added SQS long polling per suggestion by Tenzer in issue #5
Credit contributors

Author: ned

README.md

@@ -21,7 +21,7 @@ The data in CloudTrail is essential, but it's unfortunately trapped in many tiny
 * Easy to setup: under 15 minutes
 * Self-contained Kibana 3.1.2 release
 * HTTPS server with custom SSL cert/key or optional self-signed cert
-* Single Linux/OSX binaries
+* Easy-to-deploy Linux/OSX binaries, or a Docker image
 * ElasticSearch proxy ensures your logs are secure and read-only
   * No need to open direct access to your ElasticSearch instance
   * Helps to achieve PCI and HIPAA compliance in the cloud
@@ -173,5 +173,12 @@ $ make
 To cross-compile, you'll need to follow these steps first:
 http://dave.cheney.net/2012/09/08/an-introduction-to-cross-compilation-with-go
 
+## Contributors
+* [nmcclain](https://github.com/nmcclain)
+* [matthewrkrieger](https://github.com/matthewrkrieger)
+* [swindmill](https://github.com/swindmill)
+* [atward](https://github.com/atward)
+* [Tenzer](https://github.com/Tenzer)
+
 ## License
 MIT

bindata.go

@@ -176,7 +176,7 @@ func kibana_app_dashboards_default_json() (*asset, error) {
 		return nil, err
 	}
 
-	info := bindata_file_info{name: "kibana/app/dashboards/default.json", size: 8681, mode: os.FileMode(436), modTime: time.Unix(1424966498, 0)}
+	info := bindata_file_info{name: "kibana/app/dashboards/default.json", size: 8681, mode: os.FileMode(436), modTime: time.Unix(1424968096, 0)}
 	a := &asset{bytes: bytes, info:  info}
 	return a, nil
 }
@@ -2316,7 +2316,7 @@ func kibana_config_js() (*asset, error) {
 		return nil, err
 	}
 
-	info := bindata_file_info{name: "kibana/config.js", size: 2379, mode: os.FileMode(436), modTime: time.Unix(1424966498, 0)}
+	info := bindata_file_info{name: "kibana/config.js", size: 2379, mode: os.FileMode(436), modTime: time.Unix(1424968096, 0)}
 	a := &asset{bytes: bytes, info:  info}
 	return a, nil
 }

traildash.go

@@ -20,7 +20,7 @@ import (
 	"time"
 )
 
-const version = "0.8"
+const version = "0.0.9"
 
 const usage = `traildash: easy AWS CloudTrail dashboard
 
@@ -250,8 +250,7 @@ func (c *config) workLogs() {
 			kerblowie("Error dequeing from SQS: %s", err.Error())
 			continue
 		} else if m == nil {
-			log.Printf("Empty queue... sleeping for a minute.")
-			time.Sleep(60 * time.Second)
+			log.Printf("Empty queue... polling for 20 seconds.")
 			continue
 		}
 		c.debug("Fetched sqs://%s [s3://%s/%s]", m.MessageID, m.S3Bucket, m.S3ObjectKey[0])
@@ -294,6 +293,7 @@ func (c *config) dequeue() (*cloudtrailNotification, error) {
 	req := sqs.ReceiveMessageRequest{
 		QueueURL:            aws.String(c.queueURL),
 		MaxNumberOfMessages: aws.Integer(numRequested),
+		WaitTimeSeconds:     aws.Integer(20), // max allowed
 	}
 	resp, err := q.ReceiveMessage(&req)
 	if err != nil {
@@ -310,15 +310,20 @@ func (c *config) dequeue() (*cloudtrailNotification, error) {
 
 	not := sqsNotification{}
 	if err := json.Unmarshal([]byte(body), &not); err != nil {
-		return nil, fmt.Errorf("Outer JSON Unmarshal error [id: %s]: %s", not.MessageID, err.Error())
+		return nil, fmt.Errorf("SQS message JSON error [id: %s]: %s", not.MessageID, err.Error())
 	}
 
 	n := cloudtrailNotification{}
-	if err := json.Unmarshal([]byte(not.Message), &n); err != nil {
-		return nil, fmt.Errorf("Inner JSON Unmarshal error [id: %s]: %s", not.MessageID, err.Error())
-	}
 	n.MessageID = not.MessageID
 	n.ReceiptHandle = *m.ReceiptHandle
+	if not.Message == "CloudTrail validation message." { // swallow validation messages
+		if err = c.deleteSQS(&n); err != nil {
+			return nil, fmt.Errorf("Error deleting CloudTrail validation message [id: %s]: %s", not.MessageID, err.Error())
+		}
+		return nil, fmt.Errorf("Deleted CloudTrail validation message id %s", not.MessageID)
+	} else if err := json.Unmarshal([]byte(not.Message), &n); err != nil {
+		return nil, fmt.Errorf("CloudTrail JSON error [id: %s]: %s", not.MessageID, err.Error())
+	}
 	return &n, nil
 }