Merge Release/v1.2.0 to master (#267)

* updates for v1.2.0 release (#261).
* add hec-retry counter in the error message to know how many retries were needed to deliver a batch.
* updated HEC headers to include telemetry data (appname and appversion).
* update add-app-info configuration to be a list of app metadata fields.
* remove cf_origin and cf_ignored_app fields from app metadata. 
* Added memory queue pressure monitoring capability, event received and event sent count.
* add functional tests automation(#264).

Co-authored-by: Rock Baek <[email protected]>
Co-authored-by: Mei Wang <[email protected]>

Author: 3 people

.circleci/ci_nozzle_manifest.yml

@@ -1,18 +1,20 @@
 ---
 applications:
   - name: splunk-firehose-nozzle
-    memory: 256M
-    instances: 1
+    memory: 512M
+    instances: 2
     buildpack: go_buildpack
     cmd: splunk-firehose-nozzle
     env:
       GOPACKAGENAME: main
-      API_ENDPOINT: 
-      CLIENT_ID: 
-      CLIENT_SECRET: 
-      SPLUNK_HOST: 
-      SPLUNK_TOKEN: 
-      SPLUNK_INDEX: main
+      API_ENDPOINT:
+      API_USER:
+      API_PASSWORD:
+      CLIENT_ID:
+      CLIENT_SECRET:
+      SPLUNK_HOST:
+      SPLUNK_TOKEN:
+      SPLUNK_INDEX:
       SKIP_SSL_VALIDATION_CF: true
       SKIP_SSL_VALIDATION_SPLUNK: true
       JOB_NAME: splunk-nozzle
@@ -25,14 +27,13 @@ applications:
       APP_LIMITS: 1000
       BOLTDB_PATH: cache.db
       EVENTS: ValueMetric,CounterEvent,Error,LogMessage,HttpStartStop,ContainerMetric
-      EXTRA_FIELDS: name:splunk-pcf-ci
+      EXTRA_FIELDS: name:update-ci-test
       FIREHOSE_SUBSCRIPTION_ID: splunk-ci
       FLUSH_INTERVAL: 5s
       CONSUMER_QUEUE_SIZE: 10000
-      HEC_BATCH_SIZE: 100
+      HEC_BATCH_SIZE: 1000
       HEC_RETRIES: 5
       HEC_WORKERS: 8
       DEBUG: false
       ENABLE_EVENT_TRACING: true
-      RLP_GATEWAY_RETRIES: 5
-      STATUS_MONITOR_INTERVAL: 0s
+      STATUS_MONITOR_INTERVAL: 0s

.circleci/config.yml

@@ -15,7 +15,7 @@ jobs:
           name: Builder
           command: make build
       - run:
-          name: Run unit tests
+          name: Run tests
           command: |
             make testall
             cp splunk-firehose-nozzle /tmp
@@ -84,6 +84,11 @@ jobs:
             .circleci/pre-req.sh
             cf push -f .circleci/data_gen_manifest.yml -u process -p tools/data_gen --random-route
             sleep 10
+      - run:
+          name: Nozzle Log
+          command: |
+            cf logs splunk-firehose-nozzle
+          background: true
       - run:
           name: Prepare test environment
           command: |
@@ -102,37 +107,6 @@ jobs:
           when:
             always
 
-  execute_perf_tests:
-    docker:
-      - image: circleci/golang:1.12
-    working_directory: /go/src/github.com/cloudfoundry-community/splunk-firehose-nozzle
-    steps:
-      - attach_workspace:
-          at: /tmp
-      - checkout
-      - run:
-          name: Install dependencies
-          command: |
-            curl https://glide.sh/get | sh
-            go get -t ./...
-            cp -R /tmp/splunk-firehose-nozzle .
-      - run:
-          name: Prepare test environment
-          command: |
-            .circleci/pre-req.sh
-            .circleci/pre-functional-test.sh
-      - run:
-          name: Executing perf tests
-          command: |
-            .circleci/performance-test.sh
-      - run:
-          name: Teardown
-          command: |
-            echo "Teardown deployment env"
-            cf delete-org splunk-ci-org -f
-          when:
-            always
-
 workflows:
   version: 2
   build-and-deploy-nozzle:
@@ -150,25 +124,8 @@ workflows:
       - execute_tests:
           requires:
             - deploy-nozzle
-
-
-#      - execute_perf_tests:
-#          requires:
-#            - build
-#          filters:
-#            branches:
-#              only:
-#                - dev/pcf-performance-testing
-#  nightly:
-#    triggers:
-#      - schedule:
-#          cron: "0 7 * * *"
-#          filters:
-#            branches:
-#              only:
-#                - dev/pcf-performance-testing
-#    jobs:
-#      - build
-#      - execute_perf_tests:
-#          requires:
-#            - build
+          filters:
+            branches:
+              only:
+                - master
+                - release/v1.2.0

.circleci/performance-test.sh

@@ -2,21 +2,17 @@
 
 tools/dump_app_info/dump_app_info
 tools/data_gen/data_gen
-
 cache.db
 *.out
-
 main
 splunk-firehose-nozzle
 env.sh
 data_gen
-
 manifest*.yml
 splunk-firehose-nozzle.iml
-vendor/github.com/cloudfoundry/sonde-go/definitions/
-testing/integration/venv/
-testing/integration/config/local.ini
+
 *.pyc
-testing/integration/config/env.json
 testing/manual_perf/data_gen_manifest.yml
+testing/integration/config/env.json
+testing/integration/config/local.ini
 

.gitignore

@@ -67,6 +67,7 @@ test:
 
 testv:
 	@go test -v ${PKGS}
+
 # Run "short" unit tests
 test-short:
 	@go test -short ${PKGS}

Makefile

@@ -81,26 +81,24 @@ This is recommended for dev environments only.
 * `FIREHOSE_KEEP_ALIVE`: Keep alive duration for the Firehose consumer.
 * `ADD_APP_INFO`: Enrich raw data with app info. A comma separated list of app metadata (AppName,OrgName,OrgGuid,SpaceName,SpaceGuid).
 * `IGNORE_MISSING_APP`: If the application is missing, then stop repeatedly querying application info from Cloud Foundry.
-* `MISSING_APP_CACHE_INVALIDATE_TTL`:  How frequently the missing app info cache invalidates.
-* `APP_CACHE_INVALIDATE_TTL`: How frequently the app info local cache invalidates.
-* `ORG_SPACE_CACHE_INVALIDATE_TTL`: How frequently the org and space cache invalidates.
+* `MISSING_APP_CACHE_INVALIDATE_TTL`:  How frequently the missing app info cache invalidates (in s/m/h. For example, 3600s or 60m or 1h).
+* `APP_CACHE_INVALIDATE_TTL`: How frequently the app info local cache invalidates (in s/m/h. For example, 3600s or 60m or 1h).
+* `ORG_SPACE_CACHE_INVALIDATE_TTL`: How frequently the org and space cache invalidates (in s/m/h. For example, 3600s or 60m or 1h).
 * `APP_LIMITS`: Restrict to APP_LIMITS the most updated apps per request when populating the app metadata cache.
 * `BOLTDB_PATH`: Bolt database path.
 * `EVENTS`: A comma separated list of events to include. Possible values: ValueMetric,CounterEvent,Error,LogMessage,HttpStartStop,ContainerMetric
 * `EXTRA_FIELDS`: Extra fields to annotate your events with (format is key:value,key:value).
-* `FLUSH_INTERVAL`: Time interval for flushing queue to Splunk regardless of CONSUMER_QUEUE_SIZE. Protects against stale events in low throughput systems.
+* `FLUSH_INTERVAL`: Time interval (in s/m/h. For example, 3600s or 60m or 1h) for flushing queue to Splunk regardless of CONSUMER_QUEUE_SIZE. Protects against stale events in low throughput systems.
 * `CONSUMER_QUEUE_SIZE`: Sets the internal consumer queue buffer size. Events will be pushed to Splunk after queue is full.
 * `HEC_BATCH_SIZE`: Set the batch size for the events to push to HEC (Splunk HTTP Event Collector).
 * `HEC_RETRIES`: Retry count for sending events to Splunk. After expiring, events will begin dropping causing data loss.
 * `HEC_WORKERS`: Set the amount of Splunk HEC workers to increase concurrency while ingesting in Splunk.
 * `ENABLE_EVENT_TRACING`: Enables event trace logging. Splunk events will now contain a UUID, Splunk Nozzle Event Counts, and a Subscription-ID for Splunk correlation searches.
 * `SPLUNK_VERSION`: The Splunk version that determines how HEC ingests metadata fields. Only required for Splunk version 6.3 or below.
-* `RLP_GATEWAY_RETRIES`: Number of retries to connect to RLP gateway.
-* `STATUS_MONITOR_INTERVAL`: Time interval for monitoring memory queue pressure to help with back-pressure insights.
-
+* `STATUS_MONITOR_INTERVAL`: Time interval (in s/m/h. For example, 3600s or 60m or 1h) for monitoring memory queue pressure. Use to help with back-pressure insights. (Increases CPU load. Use for insights purposes only) Default is 0s (Disabled).
     ###  Please note 
     > SPLUNK_VERSION configuration parameter is only required for Splunk version 6.3 and below. 
-    For Splunk version 6.3 or below, please deploy nozzle via CLI. Update nozzle_manifest.yml with splunk_version (eg:- SPLUNK_VERSION: 6.3) as an env variable and [deploy nozzle as an app via CLI](#push-as-an-app-to-cloud-foundry).
+    For Splunk version 6.3 or below, please deploy nozzle via CLI. Update nozzle_manifest.yml with splunk_version (For example: SPLUNK_VERSION: 6.3) as an env variable and [deploy nozzle as an app via CLI](#push-as-an-app-to-cloud-foundry).
 
     **[Tile](https://network.pivotal.io/products/splunk-nozzle/)** only supports deployment for Splunk version 6.4 or above
 
@@ -137,11 +135,11 @@ on user authentication.
     ```
 
 #### Dump application info to boltdb ####
-If in production there are lots of Cloud Foundry applications(say tens of thousands) and if the user would like to enrich
-application logs by including application meta data,querying all application metadata information from Cloud Foundry may take some time.
+If in production there are lots of CF applications(say tens of thousands) and if the user would like to enrich
+application logs by including application meta data,querying all application metadata information from CF may take some time.
 For example if we include, add app name, space ID, space name, org ID and org name to the events.
 If there are multiple instances of Spunk nozzle deployed the situation will be even worse, since each of the Splunk nozzle(s) will query all applications meta data and
-cache the meta data information to the local boltdb file. These queries will introduce load to the Cloud Foundry system and could potentially take a long time to finish.
+cache the meta data information to the local boltdb file. These queries will introduce load to the CF system and could potentially take a long time to finish.
 Users can run this tool to generate a copy of all application meta data and copy this to each Splunk nozzle deployment. Each Splunk nozzle can pick up the cache copy and update the cache file incrementally afterwards.
 
 Example of how to run the dump application info tool:
@@ -239,7 +237,7 @@ This topic describes how to troubleshoot Splunk Firehose Nozzle for Cloud Foundr
 
   Are you searching for events and not finding them or looking at a dashboard and seeing "No result found"? Check Splunk Nozzle app logs.
 
-  To view the nozzle's logs running on Cloud Foundry do the following:
+  To view the nozzle's logs running on CF do the following:
 
 <ol>
   <li>Log in as an admin via the CLI.</li>
@@ -338,8 +336,8 @@ A correct setup logs a start message with configuration parameters of the Nozzle
      splunk-version: 6.6
      subscription-id: splunk-firehose
      trace-logging: true
-     rlp-gateway-retries: 5
      status-monitor-interval: 0s
+     version:
      wanted-events: ValueMetric,CounterEvent,Error,LogMessage,HttpStartStop,ContainerMetric
   }
   ip: 10.0.0.0
@@ -419,7 +417,7 @@ $ chmod +x tools/nozzle.sh
 Build project:
 
 ```
-$ make VERSION=2.0.0
+$ make VERSION=1.2.0
 ```
 
 Run tests with [Ginkgo](http://onsi.github.io/ginkgo/)

README.md

@@ -55,6 +55,10 @@ func (r *router) Route(msg *events.Envelope) error {
 		event = fevents.ErrorEvent(msg)
 	case events.Envelope_ContainerMetric:
 		event = fevents.ContainerMetric(msg)
+	case events.Envelope_HttpStart:
+		event = fevents.HttpStart(msg)
+	case events.Envelope_HttpStop:
+		event = fevents.HttpStop(msg)
 
 	default:
 		return fmt.Errorf("Unsupported event type: %s", eventType.String())

eventrouter/default.go

@@ -31,7 +31,7 @@ var _ = Describe("eventrouter", func() {
 		noCache = testing.NewMemoryCacheMock()
 		memSink = &testing.MemorySinkMock{}
 		config := &Config{
-			SelectedEvents: "LogMessage,HttpStartStop,ValueMetric,CounterEvent,Error,ContainerMetric",
+			SelectedEvents: "LogMessage,HttpStart,HttpStop,HttpStartStop,ValueMetric,CounterEvent,Error,ContainerMetric",
 		}
 		r, err = New(noCache, memSink, config)
 		Ω(err).ShouldNot(HaveOccurred())
@@ -66,7 +66,8 @@ var _ = Describe("eventrouter", func() {
 
 	It("Route valid message", func() {
 		eventTypes := []events.Envelope_EventType{
-			events.Envelope_LogMessage, events.Envelope_HttpStartStop,
+			events.Envelope_LogMessage, events.Envelope_HttpStart,
+			events.Envelope_HttpStop, events.Envelope_HttpStartStop,
 			events.Envelope_ValueMetric, events.Envelope_CounterEvent,
 			events.Envelope_Error, events.Envelope_ContainerMetric,
 		}
@@ -81,12 +82,12 @@ var _ = Describe("eventrouter", func() {
 
 	It("Route un-selected message", func() {
 		config := &Config{
-			SelectedEvents: "ValueMetric",
+			SelectedEvents: "HttpStart",
 		}
 		r, err = New(noCache, memSink, config)
 		Ω(err).ShouldNot(HaveOccurred())
 
-		eventType = events.Envelope_HttpStartStop
+		eventType = events.Envelope_HttpStop
 		err := r.Route(msg)
 		Ω(err).ShouldNot(HaveOccurred())
 		Expect(len(memSink.Events)).To(Equal(0))

eventrouter/eventrouter_test.go

@@ -35,6 +35,46 @@ var AppMetadata = []string{
 	"SpaceGuid",
 }
 
+func HttpStart(msg *events.Envelope) *Event {
+	httpStart := msg.GetHttpStart()
+	fields := logrus.Fields{
+		"timestamp":         httpStart.GetTimestamp(),
+		"request_id":        utils.FormatUUID(httpStart.GetRequestId()),
+		"method":            httpStart.GetMethod().String(),
+		"uri":               httpStart.GetUri(),
+		"remote_addr":       httpStart.GetRemoteAddress(),
+		"user_agent":        httpStart.GetUserAgent(),
+		"parent_request_id": utils.FormatUUID(httpStart.GetParentRequestId()),
+		"cf_app_id":         utils.FormatUUID(httpStart.GetApplicationId()),
+		"instance_index":    httpStart.GetInstanceIndex(),
+		"instance_id":       httpStart.GetInstanceId(),
+	}
+
+	return &Event{
+		Fields: fields,
+		Msg:    "",
+	}
+}
+
+func HttpStop(msg *events.Envelope) *Event {
+	httpStop := msg.GetHttpStop()
+
+	fields := logrus.Fields{
+		"timestamp":      httpStop.GetTimestamp(),
+		"uri":            httpStop.GetUri(),
+		"request_id":     utils.FormatUUID(httpStop.GetRequestId()),
+		"peer_type":      httpStop.GetPeerType().String(),
+		"status_code":    httpStop.GetStatusCode(),
+		"content_length": httpStop.GetContentLength(),
+		"cf_app_id":      utils.FormatUUID(httpStop.GetApplicationId()),
+	}
+
+	return &Event{
+		Fields: fields,
+		Msg:    "",
+	}
+}
+
 func HttpStartStop(msg *events.Envelope) *Event {
 	httpStartStop := msg.GetHttpStartStop()
 
@@ -159,7 +199,6 @@ func (e *Event) AnnotateWithAppData(appCache cache.Cache, config *Config) {
 		cf_space_name := appInfo.SpaceName
 		cf_org_id := appInfo.OrgGuid
 		cf_org_name := appInfo.OrgName
-		//cf_ignored_app := appInfo.IgnoredApp
 		app_env := appInfo.CfAppEnv
 
 		if cf_app_name != "" && config.AddAppName {
@@ -185,14 +224,10 @@ func (e *Event) AnnotateWithAppData(appCache cache.Cache, config *Config) {
 		if app_env["SPLUNK_INDEX"] != nil {
 			e.Fields["info_splunk_index"] = app_env["SPLUNK_INDEX"]
 		}
-		//removing cf_ignored_app as per INGEST-17639
-		//e.Fields["cf_ignored_app"] = cf_ignored_app
 	}
 }
 
 func (e *Event) AnnotateWithCFMetaData() {
-	//removing cf_origin as per INGEST-17639
-	//e.Fields["cf_origin"] = "firehose"
 	e.Fields["event_type"] = e.Type
 }