diff --git a/core/cmd/hoverfly/main.go b/core/cmd/hoverfly/main.go index 3b493a4fb..45455693d 100644 --- a/core/cmd/hoverfly/main.go +++ b/core/cmd/hoverfly/main.go @@ -64,6 +64,7 @@ var logOutputFlags arrayFlags var responseBodyFilesPath string var responseBodyFilesAllowedOriginFlags arrayFlags var journalIndexingKeyFlags arrayFlags +var journalBodyMemoryLimit util.MemorySize const boltBackend = "boltdb" const inmemoryBackend = "memory" @@ -210,6 +211,7 @@ func main() { flag.StringVar(&responseBodyFilesPath, "response-body-files-path", "", "When a response contains a relative bodyFile, it will be resolved against this absolute path (default is CWD)") flag.Var(&responseBodyFilesAllowedOriginFlags, "response-body-files-allow-origin", "When a response contains a url in bodyFile, it will be loaded only if the origin is allowed") flag.Var(&journalIndexingKeyFlags, "journal-indexing-key", "Key to setup indexing on journal") + flag.Var(&journalBodyMemoryLimit, "journal-body-memory-limit", "Memory size limit for a request or response body in the journal (e.g., '128KB', '2MB'). Memory size is unbounded by default") flag.Parse() @@ -234,6 +236,10 @@ func main() { *journalSize = 0 } + if journalBodyMemoryLimit > 0 { + log.Infof("Journal body memory limit is set to: %s", journalBodyMemoryLimit.String()) + } + if *logsSize < 0 { *logsSize = 0 } @@ -246,6 +252,7 @@ func main() { hoverfly.StoreLogsHook.LogsLimit = *logsSize hoverfly.Journal.EntryLimit = *journalSize + hoverfly.Journal.BodyMemoryLimit = journalBodyMemoryLimit // getting settings cfg := hv.InitSettings() diff --git a/core/journal/journal.go b/core/journal/journal.go index 090f32d55..8fa924a4b 100644 --- a/core/journal/journal.go +++ b/core/journal/journal.go @@ -39,10 +39,11 @@ type PostServeActionEntry struct { } type Journal struct { - entries []JournalEntry - Indexes []Index - EntryLimit int - mutex sync.Mutex + entries []JournalEntry + Indexes []Index + EntryLimit int + BodyMemoryLimit util.MemorySize + mutex sync.Mutex } func NewJournal() *Journal { @@ -106,6 +107,11 @@ func (this *Journal) NewEntry(request *http.Request, response *http.Response, mo respBody, _ := util.GetResponseBody(response) + if this.BodyMemoryLimit.ToBytes() > 0 { + payloadRequest.Body = util.TruncateStringWithEllipsis(payloadRequest.Body, this.BodyMemoryLimit.ToBytes()) + respBody = util.TruncateStringWithEllipsis(respBody, this.BodyMemoryLimit.ToBytes()) + } + payloadResponse := &models.ResponseDetails{ Status: response.StatusCode, Body: respBody, diff --git a/core/journal/journal_test.go b/core/journal/journal_test.go index 6d2a0abc5..79804a3f7 100644 --- a/core/journal/journal_test.go +++ b/core/journal/journal_test.go @@ -76,6 +76,43 @@ func Test_Journal_NewEntry_AddsJournalEntryToEntries(t *testing.T) { Expect(entries[0].Latency).To(BeNumerically("<", 1)) } +func Test_Journal_NewEntryWithMemoryLimit_TruncateBody(t *testing.T) { + RegisterTestingT(t) + + unit := journal.NewJournal() + unit.BodyMemoryLimit = 15 + + request, _ := http.NewRequest("GET", "http://hoverfly.io", io.NopCloser(bytes.NewBufferString("large request body")),) + + nowTime := time.Now() + + _, err := unit.NewEntry(request, &http.Response{ + StatusCode: 200, + Body: io.NopCloser(bytes.NewBufferString("large response body")), + Header: http.Header{ + "test-header": []string{ + "one", "two", + }, + }, + }, "test-mode", nowTime) + Expect(err).To(BeNil()) + + journalView, err := unit.GetEntries(0, 25, nil, nil, "") + entries := journalView.Journal + Expect(err).To(BeNil()) + + Expect(entries).ToNot(BeNil()) + Expect(entries).To(HaveLen(1)) + + Expect(*entries[0].Request.Method).To(Equal("GET")) + Expect(*entries[0].Request.Destination).To(Equal("hoverfly.io")) + Expect(*entries[0].Request.Body).To(Equal("large reques...")) + + Expect(entries[0].Response.Status).To(Equal(200)) + Expect(entries[0].Response.Body).To(Equal("large respon...")) +} + + func Test_Journal_UpdateEntry_AddsRemotePostServeActionToJournalEntry(t *testing.T) { RegisterTestingT(t) diff --git a/core/util/memory_size.go b/core/util/memory_size.go new file mode 100644 index 000000000..b8342b32e --- /dev/null +++ b/core/util/memory_size.go @@ -0,0 +1,53 @@ +package util + +import ( + "fmt" + "strconv" + "strings" +) + +// MemorySize is a custom type for parsing memory sizes (e.g., "128KB", "2MB") +type MemorySize int + +// String returns the string representation of the memory size +func (m *MemorySize) String() string { + return fmt.Sprintf("%d bytes", *m) +} + +// ToBytes returns the memory size as an int in bytes. +func (m *MemorySize) ToBytes() int { + return int(*m) +} + +// Set parses a string like "128KB" or "2MB" and converts it to bytes +func (m *MemorySize) Set(value string) error { + multiplier := 1 + + value = strings.ToUpper(strings.TrimSpace(value)) + + switch { + case strings.HasSuffix(value, "KB"): + multiplier = 1024 + value = strings.TrimSuffix(value, "KB") + case strings.HasSuffix(value, "MB"): + multiplier = 1024 * 1024 + value = strings.TrimSuffix(value, "MB") + case strings.HasSuffix(value, "GB"): + multiplier = 1024 * 1024 * 1024 + value = strings.TrimSuffix(value, "GB") + } + + size, err := strconv.Atoi(value) + + if err != nil { + return fmt.Errorf("invalid memory size: %s", value) + } + + if size <= 0 { + return fmt.Errorf("memory size must be greater than 0") + } + + *m = MemorySize(size * multiplier) + return nil +} + diff --git a/core/util/memory_size_test.go b/core/util/memory_size_test.go new file mode 100644 index 000000000..8f4399b61 --- /dev/null +++ b/core/util/memory_size_test.go @@ -0,0 +1,69 @@ +package util + +import "testing" +import . "github.com/onsi/gomega" + +func TestMemorySize_Set(t *testing.T) { + RegisterTestingT(t) // Register Gomega for the test + + t.Run("valid inputs", func(t *testing.T) { + var ms MemorySize + + // Test inputs with valid values + err := ms.Set("128KB") + Expect(err).To(BeNil()) + Expect(ms).To(Equal(MemorySize(128 * 1024))) + + err = ms.Set("2MB") + Expect(err).To(BeNil()) + Expect(ms).To(Equal(MemorySize(2 * 1024 * 1024))) + + err = ms.Set("1GB") + Expect(err).To(BeNil()) + Expect(ms).To(Equal(MemorySize(1 * 1024 * 1024 * 1024))) + + err = ms.Set("1024") // No suffix, treat as bytes + Expect(err).To(BeNil()) + Expect(ms).To(Equal(MemorySize(1024))) + + err = ms.Set(" 64MB ") // Test with leading/trailing spaces + Expect(err).To(BeNil()) + Expect(ms).To(Equal(MemorySize(64 * 1024 * 1024))) + }) + + t.Run("invalid inputs", func(t *testing.T) { + var ms MemorySize + + // Test inputs with invalid values + Expect(ms.Set("10XYZ")).To(MatchError("invalid memory size: 10XYZ")) + Expect(ms.Set("ABC")).To(MatchError("invalid memory size: ABC")) + Expect(ms.Set("")).To(MatchError("invalid memory size: ")) + Expect(ms.Set("-5MB")).To(MatchError("memory size must be greater than 0")) + Expect(ms).To(Equal(MemorySize(0))) + }) + + t.Run("boundary cases", func(t *testing.T) { + var ms MemorySize + + // Test extremely large numbers + err := ms.Set("1099511627776GB") // 1 PB (petabyte, very large) + Expect(err).To(BeNil()) + + // Overflow handling (in practice, you'd want to handle overflow explicitly) + err = ms.Set("9223372036854775808") // Larger than int64 max + Expect(err).To(Not(BeNil())) + }) +} + +func TestMemorySize_AsBytes(t *testing.T) { + RegisterTestingT(t) // Register Gomega for the test + + var ms MemorySize + + // Set a value and check its string representation + ms = 128 * 1024 + Expect(ms.ToBytes()).To(Equal(131072)) + + ms = 2 * 1024 * 1024 + Expect(ms.ToBytes()).To(Equal(2097152)) +} diff --git a/core/util/util.go b/core/util/util.go index 29fedd235..89420ebcf 100644 --- a/core/util/util.go +++ b/core/util/util.go @@ -22,10 +22,10 @@ import ( "reflect" "regexp" "sort" - "strings" - "strconv" + "strings" "time" + "unicode/utf8" xj "github.com/SpectoLabs/goxml2json" "github.com/tdewolff/minify/v2" @@ -557,3 +557,25 @@ func ResolveAndValidatePath(absBasePath, relativePath string) (string, error) { return resolvedPath, nil } + +func TruncateStringWithEllipsis(input string, maxSize int) string { + ellipsis := "..." // 3 bytes in UTF-8 + + if maxSize <= 3 { + return ellipsis + } + + if len(input) <= maxSize{ + return input + } + + truncated := input[:maxSize-len(ellipsis)] + + // Ensure valid UTF-8 after truncation + for !utf8.ValidString(truncated) { + truncated = truncated[:len(truncated)-1] + } + + return truncated + ellipsis +} + diff --git a/core/util/util_test.go b/core/util/util_test.go index 40c59270c..112fd843a 100644 --- a/core/util/util_test.go +++ b/core/util/util_test.go @@ -416,3 +416,73 @@ func TestResolveAndValidatePath(t *testing.T) { }) } } + + +func TestTruncateStringWithEllipsis(t *testing.T) { + + tests := []struct { + name string + input string + maxSize int + expected string + }{ + { + name: "No truncation required", + input: "Hello", + maxSize: 10, + expected: "Hello", + }, + { + name: "Zero max size", + input: "Hello", + maxSize: 0, + expected: "...", + }, + { + name: "Truncate with ellipsis", + input: "Hello, World!", + maxSize: 10, + expected: "Hello, ...", + }, + { + name: "String exactly maxSize", + input: "Hello, World!", + maxSize: 13, + expected: "Hello, World!", + }, + { + name: "Small maxSize adds ellipsis only", + input: "Hello", + maxSize: 3, + expected: "...", + }, + { + name: "UTF-8 truncation valid", + input: "你好，世界", // "Hello, World" in Chinese + maxSize: 8, + expected: "你...", + }, + { + name: "Empty string", + input: "", + maxSize: 5, + expected: "", + }, + { + name: "UTF-8 truncation with invalid byte", + input: "Hello, 世界\xef\xbf\xbd", // Invalid UTF-8 at the end + maxSize: 10, + expected: "Hello, ...", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + g := NewWithT(t) + result := TruncateStringWithEllipsis(test.input, test.maxSize) + g.Expect(result).To(Equal(test.expected), "Expected %q but got %q for input %q with maxSize %d", test.expected, result, test.input, test.maxSize) + }) + } +} + + diff --git a/functional-tests/core/ft_journal_indexing_test.go b/functional-tests/core/ft_journal_indexing_test.go index 3eda9ef88..e790a4fee 100644 --- a/functional-tests/core/ft_journal_indexing_test.go +++ b/functional-tests/core/ft_journal_indexing_test.go @@ -7,7 +7,6 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "io" - "io/ioutil" "net/http" "net/http/httptest" "strings" @@ -53,7 +52,7 @@ var _ = Describe("Manage journal indexing in hoverfly", func() { simulationResponse := hoverfly.Proxy(sling.New().Get("http://test-server.com/journaltest")) Expect(resp.StatusCode).To(Equal(200)) - body, err := ioutil.ReadAll(simulationResponse.Body) + body, err := io.ReadAll(simulationResponse.Body) Expect(err).To(BeNil()) Expect(string(body)).To(Equal("Application Testing")) @@ -98,7 +97,7 @@ var _ = Describe("Manage journal indexing in hoverfly", func() { simulationResponse := hoverfly.Proxy(sling.New().Get("http://test-server.com/journaltest")) Expect(resp.StatusCode).To(Equal(200)) - body, err := ioutil.ReadAll(simulationResponse.Body) + body, err := io.ReadAll(simulationResponse.Body) Expect(err).To(BeNil()) Expect(string(body)).To(Equal("Application Testing"))