Add memory size parser

core/cmd/hoverfly/main.go

@@ -64,6 +64,7 @@ var logOutputFlags arrayFlags
 var responseBodyFilesPath string
 var responseBodyFilesAllowedOriginFlags arrayFlags
 var journalIndexingKeyFlags arrayFlags
+var journalBodyMemoryLimit util.MemorySize
 
 const boltBackend = "boltdb"
 const inmemoryBackend = "memory"
@@ -210,6 +211,7 @@ func main() {
 	flag.StringVar(&responseBodyFilesPath, "response-body-files-path", "", "When a response contains a relative bodyFile, it will be resolved against this absolute path (default is CWD)")
 	flag.Var(&responseBodyFilesAllowedOriginFlags, "response-body-files-allow-origin", "When a response contains a url in bodyFile, it will be loaded only if the origin is allowed")
 	flag.Var(&journalIndexingKeyFlags, "journal-indexing-key", "Key to setup indexing on journal")
+	flag.Var(&journalBodyMemoryLimit, "journal-body-memory-limit", "Memory size limit for a request or response body in the journal (e.g., '128KB', '2MB'). Memory size is unbounded by default")
 
 	flag.Parse()
 
@@ -234,6 +236,10 @@ func main() {
 		*journalSize = 0
 	}
 
+	if journalBodyMemoryLimit > 0 {
+		log.Infof("Journal body memory limit is set to: %s", journalBodyMemoryLimit.String())
+	}
+
 	if *logsSize < 0 {
 		*logsSize = 0
 	}
@@ -246,6 +252,7 @@ func main() {
 
 	hoverfly.StoreLogsHook.LogsLimit = *logsSize
 	hoverfly.Journal.EntryLimit = *journalSize
+	hoverfly.Journal.BodyMemoryLimit = journalBodyMemoryLimit
 
 	// getting settings
 	cfg := hv.InitSettings()

core/journal/journal.go

@@ -39,10 +39,11 @@ type PostServeActionEntry struct {
 }
 
 type Journal struct {
-	entries    []JournalEntry
-	Indexes    []Index
-	EntryLimit int
-	mutex      sync.Mutex
+	entries         []JournalEntry
+	Indexes         []Index
+	EntryLimit      int
+	BodyMemoryLimit util.MemorySize
+	mutex           sync.Mutex
 }
 
 func NewJournal() *Journal {
@@ -106,6 +107,11 @@ func (this *Journal) NewEntry(request *http.Request, response *http.Response, mo
 
 	respBody, _ := util.GetResponseBody(response)
 
+	if this.BodyMemoryLimit.ToBytes() > 0 {
+		payloadRequest.Body = util.TruncateStringWithEllipsis(payloadRequest.Body, this.BodyMemoryLimit.ToBytes())
+		respBody = util.TruncateStringWithEllipsis(respBody, this.BodyMemoryLimit.ToBytes())
+	}
+
 	payloadResponse := &models.ResponseDetails{
 		Status:  response.StatusCode,
 		Body:    respBody,

core/journal/journal_test.go

@@ -76,6 +76,43 @@ func Test_Journal_NewEntry_AddsJournalEntryToEntries(t *testing.T) {
 	Expect(entries[0].Latency).To(BeNumerically("<", 1))
 }
 
+func Test_Journal_NewEntryWithMemoryLimit_TruncateBody(t *testing.T) {
+	RegisterTestingT(t)
+
+	unit := journal.NewJournal()
+	unit.BodyMemoryLimit = 15
+
+	request, _ := http.NewRequest("GET", "http://hoverfly.io", io.NopCloser(bytes.NewBufferString("large request body")),)
+
+	nowTime := time.Now()
+
+	_, err := unit.NewEntry(request, &http.Response{
+		StatusCode: 200,
+		Body:       io.NopCloser(bytes.NewBufferString("large response body")),
+		Header: http.Header{
+			"test-header": []string{
+				"one", "two",
+			},
+		},
+	}, "test-mode", nowTime)
+	Expect(err).To(BeNil())
+
+	journalView, err := unit.GetEntries(0, 25, nil, nil, "")
+	entries := journalView.Journal
+	Expect(err).To(BeNil())
+
+	Expect(entries).ToNot(BeNil())
+	Expect(entries).To(HaveLen(1))
+
+	Expect(*entries[0].Request.Method).To(Equal("GET"))
+	Expect(*entries[0].Request.Destination).To(Equal("hoverfly.io"))
+	Expect(*entries[0].Request.Body).To(Equal("large reques..."))
+
+	Expect(entries[0].Response.Status).To(Equal(200))
+	Expect(entries[0].Response.Body).To(Equal("large respon..."))
+}
+
+
 func Test_Journal_UpdateEntry_AddsRemotePostServeActionToJournalEntry(t *testing.T) {
 	RegisterTestingT(t)
 

core/util/memory_size.go

@@ -0,0 +1,53 @@
+package util
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+)
+
+// MemorySize is a custom type for parsing memory sizes (e.g., "128KB", "2MB")
+type MemorySize int
+
+// String returns the string representation of the memory size
+func (m *MemorySize) String() string {
+	return fmt.Sprintf("%d bytes", *m)
+}
+
+// ToBytes returns the memory size as an int in bytes.
+func (m *MemorySize) ToBytes() int {
+	return int(*m)
+}
+
+// Set parses a string like "128KB" or "2MB" and converts it to bytes
+func (m *MemorySize) Set(value string) error {
+	multiplier := 1
+
+	value = strings.ToUpper(strings.TrimSpace(value))
+
+	switch {
+	case strings.HasSuffix(value, "KB"):
+		multiplier = 1024
+		value = strings.TrimSuffix(value, "KB")
+	case strings.HasSuffix(value, "MB"):
+		multiplier = 1024 * 1024
+		value = strings.TrimSuffix(value, "MB")
+	case strings.HasSuffix(value, "GB"):
+		multiplier = 1024 * 1024 * 1024
+		value = strings.TrimSuffix(value, "GB")
+	}
+
+	size, err := strconv.Atoi(value)
+
+	if err != nil {
+		return fmt.Errorf("invalid memory size: %s", value)
+	}
+
+	if size <= 0 {
+		return fmt.Errorf("memory size must be greater than 0")
+	}
+
+	*m = MemorySize(size * multiplier)
+	return nil
+}
+

core/util/memory_size_test.go

@@ -0,0 +1,69 @@
+package util
+
+import "testing"
+import . "github.com/onsi/gomega"
+
+func TestMemorySize_Set(t *testing.T) {
+	RegisterTestingT(t) // Register Gomega for the test
+
+	t.Run("valid inputs", func(t *testing.T) {
+		var ms MemorySize
+
+		// Test inputs with valid values
+		err := ms.Set("128KB")
+		Expect(err).To(BeNil())
+		Expect(ms).To(Equal(MemorySize(128 * 1024)))
+
+		err = ms.Set("2MB")
+		Expect(err).To(BeNil())
+		Expect(ms).To(Equal(MemorySize(2 * 1024 * 1024)))
+
+		err = ms.Set("1GB")
+		Expect(err).To(BeNil())
+		Expect(ms).To(Equal(MemorySize(1 * 1024 * 1024 * 1024)))
+
+		err = ms.Set("1024") // No suffix, treat as bytes
+		Expect(err).To(BeNil())
+		Expect(ms).To(Equal(MemorySize(1024)))
+
+		err = ms.Set(" 64MB ") // Test with leading/trailing spaces
+		Expect(err).To(BeNil())
+		Expect(ms).To(Equal(MemorySize(64 * 1024 * 1024)))
+	})
+
+	t.Run("invalid inputs", func(t *testing.T) {
+		var ms MemorySize
+
+		// Test inputs with invalid values
+		Expect(ms.Set("10XYZ")).To(MatchError("invalid memory size: 10XYZ"))
+		Expect(ms.Set("ABC")).To(MatchError("invalid memory size: ABC"))
+		Expect(ms.Set("")).To(MatchError("invalid memory size: "))
+		Expect(ms.Set("-5MB")).To(MatchError("memory size must be greater than 0"))
+		Expect(ms).To(Equal(MemorySize(0)))
+	})
+
+	t.Run("boundary cases", func(t *testing.T) {
+		var ms MemorySize
+
+		// Test extremely large numbers
+		err := ms.Set("1099511627776GB") // 1 PB (petabyte, very large)
+		Expect(err).To(BeNil())
+
+		// Overflow handling (in practice, you'd want to handle overflow explicitly)
+		err = ms.Set("9223372036854775808") // Larger than int64 max
+		Expect(err).To(Not(BeNil()))
+	})
+}
+
+func TestMemorySize_AsBytes(t *testing.T) {
+	RegisterTestingT(t) // Register Gomega for the test
+
+	var ms MemorySize
+
+	// Set a value and check its string representation
+	ms = 128 * 1024
+	Expect(ms.ToBytes()).To(Equal(131072))
+
+	ms = 2 * 1024 * 1024
+	Expect(ms.ToBytes()).To(Equal(2097152))
+}

core/util/util.go

@@ -22,10 +22,10 @@ import (
 	"reflect"
 	"regexp"
 	"sort"
-	"strings"
-
 	"strconv"
+	"strings"
 	"time"
+	"unicode/utf8"
 
 	xj "github.com/SpectoLabs/goxml2json"
 	"github.com/tdewolff/minify/v2"
@@ -557,3 +557,25 @@ func ResolveAndValidatePath(absBasePath, relativePath string) (string, error) {
 
 	return resolvedPath, nil
 }
+
+func TruncateStringWithEllipsis(input string, maxSize int) string {
+	ellipsis := "..." // 3 bytes in UTF-8
+
+	if maxSize <= 3 {
+		return ellipsis
+	}
+
+	if len(input) <= maxSize{
+		return input
+	}
+
+	truncated := input[:maxSize-len(ellipsis)]
+
+	// Ensure valid UTF-8 after truncation
+	for !utf8.ValidString(truncated) {
+		truncated = truncated[:len(truncated)-1]
+	}
+
+	return truncated + ellipsis
+}
+

core/util/util_test.go

@@ -416,3 +416,73 @@ func TestResolveAndValidatePath(t *testing.T) {
 		})
 	}
 }
+
+
+func TestTruncateStringWithEllipsis(t *testing.T) {
+
+	tests := []struct {
+		name     string
+		input    string
+		maxSize  int
+		expected string
+	}{
+		{
+			name:     "No truncation required",
+			input:    "Hello",
+			maxSize:  10,
+			expected: "Hello",
+		},
+		{
+			name:     "Zero max size",
+			input:    "Hello",
+			maxSize:  0,
+			expected: "...",
+		},
+		{
+			name:     "Truncate with ellipsis",
+			input:    "Hello, World!",
+			maxSize:  10,
+			expected: "Hello, ...",
+		},
+		{
+			name:     "String exactly maxSize",
+			input:    "Hello, World!",
+			maxSize:  13,
+			expected: "Hello, World!",
+		},
+		{
+			name:     "Small maxSize adds ellipsis only",
+			input:    "Hello",
+			maxSize:  3,
+			expected: "...",
+		},
+		{
+			name:     "UTF-8 truncation valid",
+			input:    "你好，世界", // "Hello, World" in Chinese
+			maxSize:  8,
+			expected: "你...",
+		},
+		{
+			name:     "Empty string",
+			input:    "",
+			maxSize:  5,
+			expected: "",
+		},
+		{
+			name:     "UTF-8 truncation with invalid byte",
+			input:    "Hello, 世界\xef\xbf\xbd", // Invalid UTF-8 at the end
+			maxSize:  10,
+			expected: "Hello, ...",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			g := NewWithT(t)
+			result := TruncateStringWithEllipsis(test.input, test.maxSize)
+			g.Expect(result).To(Equal(test.expected), "Expected %q but got %q for input %q with maxSize %d", test.expected, result, test.input, test.maxSize)
+		})
+	}
+}
+
+

functional-tests/core/ft_journal_indexing_test.go

@@ -7,7 +7,6 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
 	"strings"
@@ -53,7 +52,7 @@ var _ = Describe("Manage journal indexing in hoverfly", func() {
 				simulationResponse := hoverfly.Proxy(sling.New().Get("http://test-server.com/journaltest"))
 				Expect(resp.StatusCode).To(Equal(200))
 
-				body, err := ioutil.ReadAll(simulationResponse.Body)
+				body, err := io.ReadAll(simulationResponse.Body)
 				Expect(err).To(BeNil())
 
 				Expect(string(body)).To(Equal("Application Testing"))
@@ -98,7 +97,7 @@ var _ = Describe("Manage journal indexing in hoverfly", func() {
 				simulationResponse := hoverfly.Proxy(sling.New().Get("http://test-server.com/journaltest"))
 				Expect(resp.StatusCode).To(Equal(200))
 
-				body, err := ioutil.ReadAll(simulationResponse.Body)
+				body, err := io.ReadAll(simulationResponse.Body)
 				Expect(err).To(BeNil())
 
 				Expect(string(body)).To(Equal("Application Testing"))