Adding log only flag

Author: krastogi

commons/src/main/scala/com/expedia/www/haystack/collector/commons/ProtoSpanExtractor.scala

@@ -142,7 +142,11 @@ class ProtoSpanExtractor(extractorConfiguration: ExtractorConfiguration,
     if (valueToValidate > highestValidValue) {
       spanSizeLimitExceededMeter.mark()
       LOGGER.debug(msg.format(span.getServiceName, span.getOperationName, span.getTraceId, valueToValidate, getProbableTagsExceedingSizeLimit(span)))
-      Success(truncateTags(span))
+      if (extractorConfiguration.spanValidation.spanMaxSize.logOnly) {
+        Success(span)
+      } else {
+        Success(truncateTags(span))
+      }
     }
     else {
       Success(span)

commons/src/main/scala/com/expedia/www/haystack/collector/commons/config/ConfigurationLoader.scala

@@ -154,6 +154,7 @@ object ConfigurationLoader {
       outputFormat = if (extractor.hasPath("output.format")) Format.withName(extractor.getString("output.format")) else Format.PROTO,
       spanValidation = SpanValidation(SpanMaxSize(
         maxSizeValidationConfig.getBoolean("enable"),
+        maxSizeValidationConfig.getBoolean("log.only"),
         maxSizeValidationConfig.getInt("max.size.limit"),
         maxSizeValidationConfig.getString("message.tag.key"),
         maxSizeValidationConfig.getString("message.tag.value"),

commons/src/main/scala/com/expedia/www/haystack/collector/commons/config/ExtractorConfiguration.scala

@@ -29,6 +29,7 @@ object Format extends Enumeration {
 case class SpanValidation(spanMaxSize: SpanMaxSize)
 
 case class SpanMaxSize(enable: Boolean,
+                       logOnly: Boolean,
                        maxSizeLimit: Int,
                        infoTagKey: String,
                        infoTagValue: String,

commons/src/test/scala/com/expedia/www/haystack/collector/commons/unit/KeyExtractorSpec.scala

@@ -36,7 +36,7 @@ class KeyExtractorSpec extends FunSpec with Matchers with MetricsSupport {
         "trace-id-1" -> createSpan("trace-id-1", "spanId_1", "service_1", "operation", StartTimeMicros, DurationMicros),
         "trace-id-2" -> createSpan("trace-id-2", "spanId_2", "service_2", "operation", StartTimeMicros, DurationMicros))
 
-      val spanValidationConfig = SpanValidation(SpanMaxSize(enable = false, 5000, "", "", Seq(), Seq()))
+      val spanValidationConfig = SpanValidation(SpanMaxSize(enable = false, logOnly = false, 5000, "", "", Seq(), Seq()))
 
       spanMap.foreach(sp => {
         val kvPairs = new ProtoSpanExtractor(ExtractorConfiguration(Format.PROTO, spanValidationConfig), LoggerFactory.getLogger(classOf[ProtoSpanExtractor]), List()).extractKeyValuePairs(sp._2.toByteArray)
@@ -54,7 +54,7 @@ class KeyExtractorSpec extends FunSpec with Matchers with MetricsSupport {
         "trace-id-1" -> createSpan("trace-id-1", "spanId_1", "service_1", "operation", StartTimeMicros, 1),
         "trace-id-2" -> createSpan("trace-id-2", "spanId_2", "service_2", "operation", StartTimeMicros, 1))
 
-      val spanValidationConfig = SpanValidation(SpanMaxSize(enable = false, 5000, "", "", Seq(), Seq()))
+      val spanValidationConfig = SpanValidation(SpanMaxSize(enable = false, logOnly = false, 5000, "", "", Seq(), Seq()))
 
       spanMap.foreach(sp => {
         val kvPairs = new ProtoSpanExtractor(ExtractorConfiguration(Format.JSON, spanValidationConfig), LoggerFactory.getLogger(classOf[ProtoSpanExtractor]), List()).extractKeyValuePairs(sp._2.toByteArray)

commons/src/test/scala/com/expedia/www/haystack/collector/commons/unit/ProtoSpanExtractorSpec.scala

@@ -34,7 +34,7 @@ class ProtoSpanExtractorSpec extends FunSpec with Matchers with MockitoSugar {
   describe("Protobuf Span Extractor") {
     val mockLogger = mock[Logger]
 
-    val spanSizeValidationConfig = SpanValidation(SpanMaxSize(enable = true, SpanSizeLimit, "X-HAYSTACK-SPAN-INFO", "Tags Truncated", Seq("error"), Seq(SkipTagTruncationServiceName)))
+    val spanSizeValidationConfig = SpanValidation(SpanMaxSize(enable = true, logOnly = false, SpanSizeLimit, "X-HAYSTACK-SPAN-INFO", "Tags Truncated", Seq("error"), Seq(SkipTagTruncationServiceName)))
     val protoSpanExtractor = new ProtoSpanExtractor(ExtractorConfiguration(Format.PROTO, spanSizeValidationConfig), mockLogger, List())
 
     val largestInvalidStartTime = SmallestAllowedStartTimeMicros - 1

http/src/main/resources/config/base.conf

@@ -13,13 +13,15 @@ extractor {
 
     # Validate size of span. Truncate span tags when size exceeds spcified limit.
     # enable: true/false
+    # log.only: if enabled, only logs such spans but doesn't truncate the tags
     # max.size.limit: maximum size allowed
     # message.tag.key: this tag key will be added when tags are truncated
     # message.tag.value: value of the above tag key indicating the truncation
     # skip.tags: truncate all span tags except these
     # skip.services: truncate span tags for all services except these
     max.size {
       enable = "false"
+      log.only = "false"
       max.size.limit = 5000 // in bytes
       message.tag.key = "X-HAYSTACK-SPAN-INFO"
       message.tag.value = "Tags are truncated. REASON: Span Size Limit Exceeded. Please contact Haystack for more details"

kinesis/src/main/resources/config/base.conf

@@ -18,13 +18,15 @@ extractor {
 
     # Validate size of span. Truncate span tags when size exceeds spcified limit.
     # enable: true/false
+    # log.only: if enabled, only logs such spans but doesn't truncate the tags
     # max.size.limit: maximum size allowed
     # message.tag.key: this tag key will be added when tags are truncated
     # message.tag.value: value of the above tag key indicating the truncation
     # skip.tags: truncate all span tags except these
     # skip.services: truncate span tags for all services except these
     max.size {
       enable = "false"
+      log.only = "false"
       max.size.limit = 5000 // in bytes
       message.tag.key = "X-HAYSTACK-SPAN-INFO"
       message.tag.value = "Tags are truncated. REASON: Span Size Limit Exceeded. Please contact Haystack for more details"

kinesis/src/test/resources/config/base.conf

@@ -15,13 +15,15 @@ extractor {
 
     # Validate size of span. Truncate span tags when size exceeds spcified limit.
     # enable: true/false
+    # log.only: if enabled, only logs such spans but doesn't truncate the tags
     # max.size.limit: maximum size allowed
     # message.tag.key: this tag key will be added when tags are truncated
     # message.tag.value: value of the above tag key indicating the truncation
     # skip.tags: truncate all span tags except these
     # skip.services: truncate span tags for all services except these
     max.size {
       enable = "false"
+      log.only = "false"
       max.size.limit = 5000 // in bytes
       message.tag.key = "X-HAYSTACK-SPAN-INFO"
       message.tag.value = "Tags are truncated. REASON: Span Size Limit Exceeded. Please contact Haystack for more details"

kinesis/src/test/scala/com/expedia/www/haystack/kinesis/span/collector/unit/tests/RecordProcessorSpec.scala

@@ -79,7 +79,7 @@ class RecordProcessorSpec extends FunSpec with Matchers with EasyMockSugar with
       }.once()
 
       whenExecuting(sink, checkpointer) {
-        val spanValidationConfig = SpanValidation(SpanMaxSize(enable = false, 5000, "", "", Seq(), Seq()))
+        val spanValidationConfig = SpanValidation(SpanMaxSize(enable = false, logOnly = false, 5000, "", "", Seq(), Seq()))
         val processor = new RecordProcessor(kinesisConfig, new ProtoSpanExtractor(ExtractorConfiguration(Format.PROTO, spanValidationConfig), LoggerFactory.getLogger(classOf[ProtoSpanExtractor]), List()), sink)
         val input = new ProcessRecordsInput().withRecords(List(record).asJava).withCheckpointer(checkpointer)
         processor.processRecords(input)
@@ -137,7 +137,7 @@ class RecordProcessorSpec extends FunSpec with Matchers with EasyMockSugar with
       }.once()
 
       whenExecuting(sink, checkpointer) {
-        val spanValidationConfig = SpanValidation(SpanMaxSize(enable = false, 5000, "", "", Seq(), Seq()))
+        val spanValidationConfig = SpanValidation(SpanMaxSize(enable = false, logOnly = false, 5000, "", "", Seq(), Seq()))
         val processor = new RecordProcessor(kinesisConfig, new ProtoSpanExtractor(ExtractorConfiguration(Format.PROTO, spanValidationConfig), LoggerFactory.getLogger(classOf[ProtoSpanExtractor]), List()), sink)
         val input_1 = new ProcessRecordsInput().withRecords(List(record_1).asJava).withCheckpointer(checkpointer)
         processor.processRecords(input_1)
@@ -166,7 +166,7 @@ class RecordProcessorSpec extends FunSpec with Matchers with EasyMockSugar with
       }.once
 
       whenExecuting(sink, checkpointer) {
-        val spanValidationConfig = SpanValidation(SpanMaxSize(enable = false, 5000, "", "", Seq(), Seq()))
+        val spanValidationConfig = SpanValidation(SpanMaxSize(enable = false, logOnly = false, 5000, "", "", Seq(), Seq()))
         val processor = new RecordProcessor(kinesisConfig, new ProtoSpanExtractor(ExtractorConfiguration(Format.PROTO, spanValidationConfig), LoggerFactory.getLogger(classOf[ProtoSpanExtractor]), List()), sink)
         val input = new ProcessRecordsInput().withRecords(List(record).asJava).withCheckpointer(checkpointer)
         processor.processRecords(input)