Merge pull request #31 from signalsciences/content-type-inspection

Improve Content-Type inspection

Author: amacnair

CHANGELOG.md

@@ -1,7 +1,8 @@
 # Golang Module Release Notes
 
-## Unreleased
+## 1.11.0 2022-01-18
 
+* Improved `Content-Type` header inspection
 * Standardized release notes
 
 ## 1.10.0 2021-05-26

LICENSE.md

@@ -2,7 +2,7 @@
 
 The MIT License (MIT)
 
-Copyright (c) 2019-2020 Signal Sciences Corp.
+Copyright (c) 2019-2022 Signal Sciences Corp.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

VERSION

@@ -1 +1 @@
-1.10.0
+1.11.0

config.go

@@ -7,6 +7,7 @@ import (
 	"net/http"
 	"path/filepath"
 	"runtime"
+	"strings"
 	"time"
 )
 
@@ -45,6 +46,7 @@ type ModuleConfig struct {
 	allowUnknownContentLength bool
 	anomalyDuration           time.Duration
 	anomalySize               int64
+	expectedContentTypes      []string
 	debug                     bool
 	headerExtractor           HeaderExtractorFunc
 	inspector                 Inspector
@@ -65,6 +67,7 @@ func NewModuleConfig(options ...ModuleConfigOption) (*ModuleConfig, error) {
 		allowUnknownContentLength: DefaultAllowUnknownContentLength,
 		anomalyDuration:           DefaultAnomalyDuration,
 		anomalySize:               DefaultAnomalySize,
+		expectedContentTypes:      make([]string, 0),
 		debug:                     DefaultDebug,
 		headerExtractor:           nil,
 		inspector:                 DefaultInspector,
@@ -108,6 +111,17 @@ func (c *ModuleConfig) IsAllowCode(code int) bool {
 	return code == 200
 }
 
+// IsExpectedContentType returns true if the given content type string is
+// in the list of configured custom Content-Types
+func (c *ModuleConfig) IsExpectedContentType(s string) bool {
+	for _, ct := range c.expectedContentTypes {
+		if strings.HasPrefix(s, ct) {
+			return true
+		}
+	}
+	return false
+}
+
 // AllowUnknownContentLength returns the configuration value
 func (c *ModuleConfig) AllowUnknownContentLength() bool {
 	return c.allowUnknownContentLength
@@ -134,6 +148,11 @@ func (c *ModuleConfig) AnomalySize() int64 {
 	return c.anomalySize
 }
 
+// ExpectedContentTypes returns the slice of additional custom content types
+func (c *ModuleConfig) ExpectedContentTypes() []string {
+	return c.expectedContentTypes
+}
+
 // Debug returns the configuration value
 func (c *ModuleConfig) Debug() bool {
 	return c.debug
@@ -250,6 +269,15 @@ func AnomalySize(size int64) ModuleConfigOption {
 	}
 }
 
+// ExpectedContentType is a function argument that adds a custom Content-Type
+// that should have request bodies sent to the agent for inspection
+func ExpectedContentType(s string) ModuleConfigOption {
+	return func(c *ModuleConfig) error {
+		c.expectedContentTypes = append(c.expectedContentTypes, s)
+		return nil
+	}
+}
+
 // CustomInspector is a function argument that sets a custom inspector,
 // an optional inspector initializer to decide if inspection should occur, and
 // an optional inspector finalizer that can perform any post-inspection steps

config_test.go

@@ -23,6 +23,9 @@ func TestDefaultModuleConfig(t *testing.T) {
 	if c.AnomalySize() != DefaultAnomalySize {
 		t.Errorf("Unexpected AnomalySize: %v", c.AnomalySize())
 	}
+	if len(c.ExpectedContentTypes()) != 0 {
+		t.Errorf("Unexpected ExpectedContentTypes: expected length 0, got %d", len(c.ExpectedContentTypes()))
+	}
 	if c.Debug() != DefaultDebug {
 		t.Errorf("Unexpected Debug: %v", c.Debug())
 	}
@@ -88,6 +91,8 @@ func TestConfiguredModuleConfig(t *testing.T) {
 		AnomalySize(8192),
 		CustomInspector(&RPCInspector{}, func(_ *http.Request) bool { return true }, func(_ *http.Request) {}),
 		CustomHeaderExtractor(func(_ *http.Request) (http.Header, error) { return nil, nil }),
+		ExpectedContentType("application/foobar"),
+		ExpectedContentType("application/fizzbuzz"),
 		Debug(true),
 		MaxContentLength(500000),
 		Socket("tcp", "0.0.0.0:1234"),
@@ -108,6 +113,9 @@ func TestConfiguredModuleConfig(t *testing.T) {
 	if c.AnomalySize() != 8192 {
 		t.Errorf("Unexpected AnomalySize: %v", c.AnomalySize())
 	}
+	if len(c.ExpectedContentTypes()) != 2 || c.ExpectedContentTypes()[0] != "application/foobar" || c.ExpectedContentTypes()[1] != "application/fizzbuzz" {
+		t.Errorf("Unexpected ExpectedContentTypes: %v", c.ExpectedContentTypes())
+	}
 	if c.Debug() != true {
 		t.Errorf("Unexpected Debug: %v", c.Debug())
 	}
@@ -171,6 +179,8 @@ func TestFromModuleConfig(t *testing.T) {
 		AltResponseCodes(403),
 		AnomalyDuration(10*time.Second),
 		AnomalySize(8192),
+		ExpectedContentType("application/foobar"),
+		ExpectedContentType("application/fizzbuzz"),
 		CustomInspector(&RPCInspector{}, func(_ *http.Request) bool { return true }, func(_ *http.Request) {}),
 		CustomHeaderExtractor(func(_ *http.Request) (http.Header, error) { return nil, nil }),
 		Debug(true),
@@ -200,6 +210,9 @@ func TestFromModuleConfig(t *testing.T) {
 	if c.AnomalySize() != 8192 {
 		t.Errorf("Unexpected AnomalySize: %v", c.AnomalySize())
 	}
+	if len(c.ExpectedContentTypes()) != 2 || c.ExpectedContentTypes()[0] != "application/foobar" || c.ExpectedContentTypes()[1] != "application/fizzbuzz" {
+		t.Errorf("Unexpected ExpectedContentTypes: %v", c.ExpectedContentTypes())
+	}
 	if c.Debug() != true {
 		t.Errorf("Unexpected Debug: %v", c.Debug())
 	}

module.go

@@ -445,7 +445,26 @@ func shouldReadBody(req *http.Request, m *Module) bool {
 	}
 
 	// only read certain types of content
-	return inspectableContentType(req.Header.Get("Content-Type"))
+	if inspectableContentType(req.Header.Get("Content-Type")) {
+		return true
+	}
+
+	// read custom configured content type(s)
+	if m.config.IsExpectedContentType(req.Header.Get("Content-Type")) {
+		return true
+	}
+
+	// read the body if there are multiple Content-Type headers
+	if len(req.Header.Values("Content-Type")) > 1 {
+		return true
+	}
+
+	// Check for comma separated Content-Types
+	if len(strings.SplitN(req.Header.Get("Content-Type"), ",", 2)) > 1 {
+		return true
+	}
+
+	return false
 }
 
 // inspectableContentType returns true for an inspectable content type
@@ -477,6 +496,10 @@ func inspectableContentType(s string) bool {
 	// GraphQL
 	case strings.HasPrefix(s, "application/graphql"):
 		return true
+
+	// No type provided
+	case s == "":
+		return true
 	}
 
 	return false

module_test.go

@@ -271,6 +271,7 @@ func TestInspectableContentType(t *testing.T) {
 		{true, "text/x-json"},
 		{true, "application/javascript"},
 		{true, "application/graphql"},
+		{true, ""},
 		{false, "octet/stream"},
 		{false, "junk/yard"},
 	}
@@ -364,6 +365,7 @@ func TestModule(t *testing.T) {
 		w := httptest.NewRecorder()
 		m.ServeHTTP(w, req)
 		resp := w.Result()
+		defer resp.Body.Close()
 
 		if dump, err := httputil.DumpRequest(req, true); err == nil {
 			t.Log("SERVER REQUEST:\n" + string(dump))

responsewriter_test.go

@@ -87,6 +87,8 @@ func testResponseWriter(t *testing.T, w ResponseWriter, flusher bool) {
 
 	// Verify the response
 	resp := recorder.Result()
+	defer resp.Body.Close()
+
 	if resp.StatusCode != status {
 		t.Errorf("Unexpected status code=%d, expected=%d", resp.StatusCode, status)
 	}

version.go

@@ -1,3 +1,3 @@
 package sigsci
 
-const version = "1.10.0"
+const version = "1.11.0"