From e4a3ae9b5dba30119851801afb1b0efce11724e8 Mon Sep 17 00:00:00 2001 From: Tommy Situ Date: Sun, 2 Mar 2025 22:00:40 +0000 Subject: [PATCH 1/6] Add memory size parser --- core/util/memory_size.go | 48 +++++++++++++++++++++++++ core/util/memory_size_test.go | 68 +++++++++++++++++++++++++++++++++++ 2 files changed, 116 insertions(+) create mode 100644 core/util/memory_size.go create mode 100644 core/util/memory_size_test.go diff --git a/core/util/memory_size.go b/core/util/memory_size.go new file mode 100644 index 000000000..4310e6f75 --- /dev/null +++ b/core/util/memory_size.go @@ -0,0 +1,48 @@ +package util + +import ( + "fmt" + "strconv" + "strings" +) + +// MemorySize is a custom type for parsing memory sizes (e.g., "128KB", "2MB") +type MemorySize int64 + +// String returns the string representation of the memory size +func (m *MemorySize) String() string { + return fmt.Sprintf("%d", *m) +} + +// ToBytes returns the memory size as an int64 in bytes. +func (m *MemorySize) ToBytes() int64 { + return int64(*m) +} + +// Set parses a string like "128KB" or "2MB" and converts it to bytes +func (m *MemorySize) Set(value string) error { + multiplier := int64(1) + + value = strings.ToUpper(strings.TrimSpace(value)) + + switch { + case strings.HasSuffix(value, "KB"): + multiplier = 1024 + value = strings.TrimSuffix(value, "KB") + case strings.HasSuffix(value, "MB"): + multiplier = 1024 * 1024 + value = strings.TrimSuffix(value, "MB") + case strings.HasSuffix(value, "GB"): + multiplier = 1024 * 1024 * 1024 + value = strings.TrimSuffix(value, "GB") + } + + size, err := strconv.ParseInt(value, 10, 64) + if err != nil || size < 0 { + return fmt.Errorf("invalid memory size: %s", value) + } + + *m = MemorySize(size * multiplier) + return nil +} + diff --git a/core/util/memory_size_test.go b/core/util/memory_size_test.go new file mode 100644 index 000000000..db9ba7d06 --- /dev/null +++ b/core/util/memory_size_test.go @@ -0,0 +1,68 @@ +package util + +import "testing" +import . "github.com/onsi/gomega" + +func TestMemorySize_Set(t *testing.T) { + RegisterTestingT(t) // Register Gomega for the test + + t.Run("valid inputs", func(t *testing.T) { + var ms MemorySize + + // Test inputs with valid values + err := ms.Set("128KB") + Expect(err).To(BeNil()) + Expect(ms).To(Equal(MemorySize(128 * 1024))) + + err = ms.Set("2MB") + Expect(err).To(BeNil()) + Expect(ms).To(Equal(MemorySize(2 * 1024 * 1024))) + + err = ms.Set("1GB") + Expect(err).To(BeNil()) + Expect(ms).To(Equal(MemorySize(1 * 1024 * 1024 * 1024))) + + err = ms.Set("1024") // No suffix, treat as bytes + Expect(err).To(BeNil()) + Expect(ms).To(Equal(MemorySize(1024))) + + err = ms.Set(" 64MB ") // Test with leading/trailing spaces + Expect(err).To(BeNil()) + Expect(ms).To(Equal(MemorySize(64 * 1024 * 1024))) + }) + + t.Run("invalid inputs", func(t *testing.T) { + var ms MemorySize + + // Test inputs with invalid values + Expect(ms.Set("10XYZ")).To(Not(BeNil())) // Unknown unit + Expect(ms.Set("ABC")).To(Not(BeNil())) // Non-numeric input + Expect(ms.Set("")).To(Not(BeNil())) // Empty input + Expect(ms.Set("-5MB")).To(Not(BeNil())) // Negative value + }) + + t.Run("boundary cases", func(t *testing.T) { + var ms MemorySize + + // Test extremely large numbers + err := ms.Set("1099511627776GB") // 1 PB (petabyte, very large) + Expect(err).To(BeNil()) + + // Overflow handling (in practice, you'd want to handle overflow explicitly) + err = ms.Set("9223372036854775808") // Larger than int64 max + Expect(err).To(Not(BeNil())) + }) +} + +func TestMemorySize_AsBytes(t *testing.T) { + RegisterTestingT(t) // Register Gomega for the test + + var ms MemorySize + + // Set a value and check its string representation + ms = 128 * 1024 + Expect(ms.ToBytes()).To(Equal(int64(131072))) + + ms = 2 * 1024 * 1024 + Expect(ms.ToBytes()).To(Equal(int64(2097152))) +} From 85c9f41c4284021d8ac25e995941473da654104b Mon Sep 17 00:00:00 2001 From: Tommy Situ Date: Sun, 2 Mar 2025 22:19:39 +0000 Subject: [PATCH 2/6] Add journal body memory limit flag --- core/cmd/hoverfly/main.go | 6 ++++++ core/util/memory_size.go | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/core/cmd/hoverfly/main.go b/core/cmd/hoverfly/main.go index 3b493a4fb..2d9995d6d 100644 --- a/core/cmd/hoverfly/main.go +++ b/core/cmd/hoverfly/main.go @@ -64,6 +64,7 @@ var logOutputFlags arrayFlags var responseBodyFilesPath string var responseBodyFilesAllowedOriginFlags arrayFlags var journalIndexingKeyFlags arrayFlags +var journalBodyMemoryLimit util.MemorySize const boltBackend = "boltdb" const inmemoryBackend = "memory" @@ -210,6 +211,7 @@ func main() { flag.StringVar(&responseBodyFilesPath, "response-body-files-path", "", "When a response contains a relative bodyFile, it will be resolved against this absolute path (default is CWD)") flag.Var(&responseBodyFilesAllowedOriginFlags, "response-body-files-allow-origin", "When a response contains a url in bodyFile, it will be loaded only if the origin is allowed") flag.Var(&journalIndexingKeyFlags, "journal-indexing-key", "Key to setup indexing on journal") + flag.Var(&journalBodyMemoryLimit, "journal-body-memory-limit", "Memory size limit for a request or response body in the journal (e.g., '128KB', '2MB'). Memory size is unbounded by default") flag.Parse() @@ -234,6 +236,10 @@ func main() { *journalSize = 0 } + if journalBodyMemoryLimit > 0 { + log.Infof("Journal body memory limit is set to: %s", journalBodyMemoryLimit.String()) + } + if *logsSize < 0 { *logsSize = 0 } diff --git a/core/util/memory_size.go b/core/util/memory_size.go index 4310e6f75..5fbed6e3b 100644 --- a/core/util/memory_size.go +++ b/core/util/memory_size.go @@ -11,7 +11,7 @@ type MemorySize int64 // String returns the string representation of the memory size func (m *MemorySize) String() string { - return fmt.Sprintf("%d", *m) + return fmt.Sprintf("%d bytes", *m) } // ToBytes returns the memory size as an int64 in bytes. From 096d52390cc2400d06459cfb1d2c3c46ded4b325 Mon Sep 17 00:00:00 2001 From: Tommy Situ Date: Sun, 2 Mar 2025 22:55:32 +0000 Subject: [PATCH 3/6] Add truncate string util function --- core/cmd/hoverfly/main.go | 1 + core/journal/journal.go | 9 ++--- core/util/memory_size.go | 12 +++---- core/util/memory_size_test.go | 4 +-- core/util/util.go | 22 ++++++++++-- core/util/util_test.go | 64 +++++++++++++++++++++++++++++++++++ 6 files changed, 98 insertions(+), 14 deletions(-) diff --git a/core/cmd/hoverfly/main.go b/core/cmd/hoverfly/main.go index 2d9995d6d..45455693d 100644 --- a/core/cmd/hoverfly/main.go +++ b/core/cmd/hoverfly/main.go @@ -252,6 +252,7 @@ func main() { hoverfly.StoreLogsHook.LogsLimit = *logsSize hoverfly.Journal.EntryLimit = *journalSize + hoverfly.Journal.BodyMemoryLimit = journalBodyMemoryLimit // getting settings cfg := hv.InitSettings() diff --git a/core/journal/journal.go b/core/journal/journal.go index 090f32d55..646da1e00 100644 --- a/core/journal/journal.go +++ b/core/journal/journal.go @@ -39,10 +39,11 @@ type PostServeActionEntry struct { } type Journal struct { - entries []JournalEntry - Indexes []Index - EntryLimit int - mutex sync.Mutex + entries []JournalEntry + Indexes []Index + EntryLimit int + BodyMemoryLimit util.MemorySize + mutex sync.Mutex } func NewJournal() *Journal { diff --git a/core/util/memory_size.go b/core/util/memory_size.go index 5fbed6e3b..4640b2e75 100644 --- a/core/util/memory_size.go +++ b/core/util/memory_size.go @@ -7,21 +7,21 @@ import ( ) // MemorySize is a custom type for parsing memory sizes (e.g., "128KB", "2MB") -type MemorySize int64 +type MemorySize int // String returns the string representation of the memory size func (m *MemorySize) String() string { return fmt.Sprintf("%d bytes", *m) } -// ToBytes returns the memory size as an int64 in bytes. -func (m *MemorySize) ToBytes() int64 { - return int64(*m) +// ToBytes returns the memory size as an int in bytes. +func (m *MemorySize) ToBytes() int { + return int(*m) } // Set parses a string like "128KB" or "2MB" and converts it to bytes func (m *MemorySize) Set(value string) error { - multiplier := int64(1) + multiplier := 1 value = strings.ToUpper(strings.TrimSpace(value)) @@ -37,7 +37,7 @@ func (m *MemorySize) Set(value string) error { value = strings.TrimSuffix(value, "GB") } - size, err := strconv.ParseInt(value, 10, 64) + size, err := strconv.Atoi(value) if err != nil || size < 0 { return fmt.Errorf("invalid memory size: %s", value) } diff --git a/core/util/memory_size_test.go b/core/util/memory_size_test.go index db9ba7d06..52603af3e 100644 --- a/core/util/memory_size_test.go +++ b/core/util/memory_size_test.go @@ -61,8 +61,8 @@ func TestMemorySize_AsBytes(t *testing.T) { // Set a value and check its string representation ms = 128 * 1024 - Expect(ms.ToBytes()).To(Equal(int64(131072))) + Expect(ms.ToBytes()).To(Equal(131072)) ms = 2 * 1024 * 1024 - Expect(ms.ToBytes()).To(Equal(int64(2097152))) + Expect(ms.ToBytes()).To(Equal(2097152)) } diff --git a/core/util/util.go b/core/util/util.go index 29fedd235..0022150ce 100644 --- a/core/util/util.go +++ b/core/util/util.go @@ -22,10 +22,10 @@ import ( "reflect" "regexp" "sort" - "strings" - "strconv" + "strings" "time" + "unicode/utf8" xj "github.com/SpectoLabs/goxml2json" "github.com/tdewolff/minify/v2" @@ -557,3 +557,21 @@ func ResolveAndValidatePath(absBasePath, relativePath string) (string, error) { return resolvedPath, nil } + +func truncateStringWithEllipsis(input string, maxSize int) string { + ellipsis := "..." + + if len(input) <= maxSize{ + return input + } + + truncated := input[:maxSize-len(ellipsis)] + + // Ensure valid UTF-8 after truncation + for !utf8.ValidString(truncated) { + truncated = truncated[:len(truncated)-1] + } + + return truncated + ellipsis +} + diff --git a/core/util/util_test.go b/core/util/util_test.go index 40c59270c..e470f8ef5 100644 --- a/core/util/util_test.go +++ b/core/util/util_test.go @@ -416,3 +416,67 @@ func TestResolveAndValidatePath(t *testing.T) { }) } } + + +func TestTruncateStringWithEllipsis(t *testing.T) { + + tests := []struct { + name string + input string + maxSize int + expected string + }{ + { + name: "No truncation required", + input: "Hello", + maxSize: 10, + expected: "Hello", + }, + { + name: "Truncate with ellipsis", + input: "Hello, World!", + maxSize: 10, + expected: "Hello, ...", + }, + { + name: "String exactly maxSize", + input: "Hello, World!", + maxSize: 13, + expected: "Hello, World!", + }, + { + name: "Small maxSize adds ellipsis only", + input: "Hello", + maxSize: 3, + expected: "...", + }, + { + name: "UTF-8 truncation valid", + input: "你好，世界", // "Hello, World" in Chinese + maxSize: 8, + expected: "你...", + }, + { + name: "Empty string", + input: "", + maxSize: 5, + expected: "", + }, + { + name: "UTF-8 truncation with invalid byte", + input: "Hello, 世界\xef\xbf\xbd", // Invalid UTF-8 at the end + maxSize: 10, + expected: "Hello, ...", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + g := NewWithT(t) + result := truncateStringWithEllipsis(test.input, test.maxSize) + g.Expect(result).To(Equal(test.expected), "Expected %q but got %q for input %q with maxSize %d", test.expected, result, test.input, test.maxSize) + }) + } +} + + From e5d64b910635a96ea0b7c90f8a9f8489df92a55e Mon Sep 17 00:00:00 2001 From: Tommy Situ Date: Sun, 2 Mar 2025 23:16:25 +0000 Subject: [PATCH 4/6] Journal body memory limit should be greater than zero --- core/util/memory_size.go | 7 ++++++- core/util/memory_size_test.go | 9 +++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/core/util/memory_size.go b/core/util/memory_size.go index 4640b2e75..b8342b32e 100644 --- a/core/util/memory_size.go +++ b/core/util/memory_size.go @@ -38,10 +38,15 @@ func (m *MemorySize) Set(value string) error { } size, err := strconv.Atoi(value) - if err != nil || size < 0 { + + if err != nil { return fmt.Errorf("invalid memory size: %s", value) } + if size <= 0 { + return fmt.Errorf("memory size must be greater than 0") + } + *m = MemorySize(size * multiplier) return nil } diff --git a/core/util/memory_size_test.go b/core/util/memory_size_test.go index 52603af3e..8f4399b61 100644 --- a/core/util/memory_size_test.go +++ b/core/util/memory_size_test.go @@ -35,10 +35,11 @@ func TestMemorySize_Set(t *testing.T) { var ms MemorySize // Test inputs with invalid values - Expect(ms.Set("10XYZ")).To(Not(BeNil())) // Unknown unit - Expect(ms.Set("ABC")).To(Not(BeNil())) // Non-numeric input - Expect(ms.Set("")).To(Not(BeNil())) // Empty input - Expect(ms.Set("-5MB")).To(Not(BeNil())) // Negative value + Expect(ms.Set("10XYZ")).To(MatchError("invalid memory size: 10XYZ")) + Expect(ms.Set("ABC")).To(MatchError("invalid memory size: ABC")) + Expect(ms.Set("")).To(MatchError("invalid memory size: ")) + Expect(ms.Set("-5MB")).To(MatchError("memory size must be greater than 0")) + Expect(ms).To(Equal(MemorySize(0))) }) t.Run("boundary cases", func(t *testing.T) { From 32ceebb434abae1d9860cf2aa0dc7688e8290275 Mon Sep 17 00:00:00 2001 From: Tommy Situ Date: Sun, 2 Mar 2025 23:33:56 +0000 Subject: [PATCH 5/6] Truncate request response body if journal memory limit is set --- core/journal/journal.go | 7 ++++++- core/util/util.go | 8 ++++++-- core/util/util_test.go | 8 +++++++- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/core/journal/journal.go b/core/journal/journal.go index 646da1e00..a7a28c319 100644 --- a/core/journal/journal.go +++ b/core/journal/journal.go @@ -107,9 +107,14 @@ func (this *Journal) NewEntry(request *http.Request, response *http.Response, mo respBody, _ := util.GetResponseBody(response) + if this.BodyMemoryLimit.ToBytes() > 0 { + payloadRequest.Body = util.TruncateStringWithEllipsis(payloadRequest.Body, this.BodyMemoryLimit.ToBytes()) + respBody = util.TruncateStringWithEllipsis(respBody, this.BodyMemoryLimit.ToBytes()) + } + payloadResponse := &models.ResponseDetails{ Status: response.StatusCode, - Body: respBody, + Body: util.TruncateStringWithEllipsis(respBody, this.BodyMemoryLimit.ToBytes()), Headers: response.Header, } diff --git a/core/util/util.go b/core/util/util.go index 0022150ce..89420ebcf 100644 --- a/core/util/util.go +++ b/core/util/util.go @@ -558,8 +558,12 @@ func ResolveAndValidatePath(absBasePath, relativePath string) (string, error) { return resolvedPath, nil } -func truncateStringWithEllipsis(input string, maxSize int) string { - ellipsis := "..." +func TruncateStringWithEllipsis(input string, maxSize int) string { + ellipsis := "..." // 3 bytes in UTF-8 + + if maxSize <= 3 { + return ellipsis + } if len(input) <= maxSize{ return input diff --git a/core/util/util_test.go b/core/util/util_test.go index e470f8ef5..112fd843a 100644 --- a/core/util/util_test.go +++ b/core/util/util_test.go @@ -432,6 +432,12 @@ func TestTruncateStringWithEllipsis(t *testing.T) { maxSize: 10, expected: "Hello", }, + { + name: "Zero max size", + input: "Hello", + maxSize: 0, + expected: "...", + }, { name: "Truncate with ellipsis", input: "Hello, World!", @@ -473,7 +479,7 @@ func TestTruncateStringWithEllipsis(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { g := NewWithT(t) - result := truncateStringWithEllipsis(test.input, test.maxSize) + result := TruncateStringWithEllipsis(test.input, test.maxSize) g.Expect(result).To(Equal(test.expected), "Expected %q but got %q for input %q with maxSize %d", test.expected, result, test.input, test.maxSize) }) } From a18af5ca04d0b2fecf787be281902f687344d682 Mon Sep 17 00:00:00 2001 From: Tommy Situ Date: Sun, 2 Mar 2025 23:47:39 +0000 Subject: [PATCH 6/6] Add unit test for journal entry body truncation --- core/journal/journal.go | 2 +- core/journal/journal_test.go | 37 +++++++++++++++++++ .../core/ft_journal_indexing_test.go | 5 +-- 3 files changed, 40 insertions(+), 4 deletions(-) diff --git a/core/journal/journal.go b/core/journal/journal.go index a7a28c319..8fa924a4b 100644 --- a/core/journal/journal.go +++ b/core/journal/journal.go @@ -114,7 +114,7 @@ func (this *Journal) NewEntry(request *http.Request, response *http.Response, mo payloadResponse := &models.ResponseDetails{ Status: response.StatusCode, - Body: util.TruncateStringWithEllipsis(respBody, this.BodyMemoryLimit.ToBytes()), + Body: respBody, Headers: response.Header, } diff --git a/core/journal/journal_test.go b/core/journal/journal_test.go index 6d2a0abc5..79804a3f7 100644 --- a/core/journal/journal_test.go +++ b/core/journal/journal_test.go @@ -76,6 +76,43 @@ func Test_Journal_NewEntry_AddsJournalEntryToEntries(t *testing.T) { Expect(entries[0].Latency).To(BeNumerically("<", 1)) } +func Test_Journal_NewEntryWithMemoryLimit_TruncateBody(t *testing.T) { + RegisterTestingT(t) + + unit := journal.NewJournal() + unit.BodyMemoryLimit = 15 + + request, _ := http.NewRequest("GET", "http://hoverfly.io", io.NopCloser(bytes.NewBufferString("large request body")),) + + nowTime := time.Now() + + _, err := unit.NewEntry(request, &http.Response{ + StatusCode: 200, + Body: io.NopCloser(bytes.NewBufferString("large response body")), + Header: http.Header{ + "test-header": []string{ + "one", "two", + }, + }, + }, "test-mode", nowTime) + Expect(err).To(BeNil()) + + journalView, err := unit.GetEntries(0, 25, nil, nil, "") + entries := journalView.Journal + Expect(err).To(BeNil()) + + Expect(entries).ToNot(BeNil()) + Expect(entries).To(HaveLen(1)) + + Expect(*entries[0].Request.Method).To(Equal("GET")) + Expect(*entries[0].Request.Destination).To(Equal("hoverfly.io")) + Expect(*entries[0].Request.Body).To(Equal("large reques...")) + + Expect(entries[0].Response.Status).To(Equal(200)) + Expect(entries[0].Response.Body).To(Equal("large respon...")) +} + + func Test_Journal_UpdateEntry_AddsRemotePostServeActionToJournalEntry(t *testing.T) { RegisterTestingT(t) diff --git a/functional-tests/core/ft_journal_indexing_test.go b/functional-tests/core/ft_journal_indexing_test.go index 3eda9ef88..e790a4fee 100644 --- a/functional-tests/core/ft_journal_indexing_test.go +++ b/functional-tests/core/ft_journal_indexing_test.go @@ -7,7 +7,6 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "io" - "io/ioutil" "net/http" "net/http/httptest" "strings" @@ -53,7 +52,7 @@ var _ = Describe("Manage journal indexing in hoverfly", func() { simulationResponse := hoverfly.Proxy(sling.New().Get("http://test-server.com/journaltest")) Expect(resp.StatusCode).To(Equal(200)) - body, err := ioutil.ReadAll(simulationResponse.Body) + body, err := io.ReadAll(simulationResponse.Body) Expect(err).To(BeNil()) Expect(string(body)).To(Equal("Application Testing")) @@ -98,7 +97,7 @@ var _ = Describe("Manage journal indexing in hoverfly", func() { simulationResponse := hoverfly.Proxy(sling.New().Get("http://test-server.com/journaltest")) Expect(resp.StatusCode).To(Equal(200)) - body, err := ioutil.ReadAll(simulationResponse.Body) + body, err := io.ReadAll(simulationResponse.Body) Expect(err).To(BeNil()) Expect(string(body)).To(Equal("Application Testing"))