updates based off review

Author: gregns1

base/constants.go

@@ -155,6 +155,9 @@ const (
 
 	// FromConnStrWarningThreshold determines the amount of time it should take before we warn about parsing a connstr (mostly for DNS resolution)
 	FromConnStrWarningThreshold = 10 * time.Second
+
+	// StackFilePrefix is the prefix used when writing stack trace files
+	StackFilePrefix = "sg_stack_trace_"
 )
 
 // SyncGatewayRawDocXattrs is a list of xattrs that Sync Gateway will fetch when reading a raw document.

base/util.go

@@ -33,6 +33,7 @@ import (
 	"regexp"
 	"runtime"
 	"runtime/debug"
+	"runtime/pprof"
 	"slices"
 	"sort"
 	"strconv"
@@ -1835,19 +1836,10 @@ func IsRevTreeID(s string) bool {
 }
 
 // GetStackTrace will return goroutine stack traces for all goroutines in Sync Gateway.
-func GetStackTrace() string {
-	// make 1MB buffer but if this buffer isn't big enough, runtime.Stack will
-	// return nothing, thus have 5 retires doubling the capacity each time.
-	buf := make([]byte, 1<<20)
-	for range 5 {
-		n := runtime.Stack(buf, true)
-		if n < len(buf) {
-			buf = buf[:n]
-			break
-		}
-		buf = make([]byte, 2*len(buf))
-	}
-	return string(buf)
+func GetStackTrace() (bytes.Buffer, error) {
+	profBuf := bytes.Buffer{}
+	err := pprof.Lookup("goroutine").WriteTo(&profBuf, 2)
+	return profBuf, err
 }
 
 // RotateProfilesIfNeeded will remove old files if there are more than
@@ -1870,3 +1862,32 @@ func RotateProfilesIfNeeded(filename string) error {
 	}
 	return multiErr.ErrorOrNil()
 }
+
+func LogStackTraces(ctx context.Context, logDirectory string, stackTrace bytes.Buffer, timestamp string) {
+
+	// log to console
+	_, _ = fmt.Fprintf(os.Stderr, "Stack trace:\n%s\n", stackTrace.String())
+
+	filename := filepath.Join(logDirectory, StackFilePrefix+timestamp+".log")
+	file, err := os.Create(filename)
+	defer func() {
+		closeErr := file.Close()
+		if closeErr != nil {
+			WarnfCtx(ctx, "Error closing stack trace file %s: %v", filename, closeErr)
+		}
+	}()
+	if err != nil {
+		WarnfCtx(ctx, "Error opening stack trace file %s: %v", filename, err)
+	}
+
+	_, err = file.WriteString(fmt.Sprintf("Stack trace:\n%s\n", stackTrace.String()))
+	if err != nil {
+		WarnfCtx(ctx, "Error writing stack trace to file %s: %v", filename, err)
+	}
+
+	rotatePath := filepath.Join(logDirectory, StackFilePrefix+"*.log")
+	err = RotateProfilesIfNeeded(rotatePath)
+	if err != nil {
+		WarnfCtx(ctx, "Error rotating stack trace files in path %s: %v", rotatePath, err)
+	}
+}

docs/api/admin.yaml

@@ -97,8 +97,6 @@ paths:
     $ref: ./paths/admin/_debug-pprof-trace.yaml
   /_debug/fgprof:
     $ref: ./paths/admin/_debug-fgprof.yaml
-  /_debug/stacktrace:
-    $ref: ./paths/admin/_debug-stack_trace.yaml
   /_post_upgrade:
     $ref: ./paths/admin/_post_upgrade.yaml
   '/{db}/_config':

docs/api/paths/admin/_debug-stack_trace.yaml

@@ -2273,11 +2273,11 @@ func TestHandleGetStackTrace(t *testing.T) {
 	rt := rest.NewRestTester(t, nil)
 	defer rt.Close()
 
-	resp := rt.SendAdminRequest(http.MethodGet, "/_debug/stacktrace", "")
+	resp := rt.SendAdminRequest(http.MethodGet, "/_debug/pprof/goroutine?debug=2", "")
 	rest.RequireStatus(t, resp, http.StatusOK)
 	rawResponseStr := resp.Body.String()
 	assert.Contains(t, rawResponseStr, "goroutine")
-	assert.Contains(t, rawResponseStr, "handleCollectStackTrace")
+	assert.Contains(t, rawResponseStr, "handlePprofGoroutine")
 }
 
 func TestConfigRedaction(t *testing.T) {

rest/adminapitest/admin_api_test.go

@@ -693,14 +693,6 @@ func (h *handler) handleFgprof() error {
 	return stopFn()
 }
 
-func (h *handler) handleCollectStackTrace() error {
-
-	stackTrace := base.GetStackTrace()
-
-	h.writeText([]byte(stackTrace))
-	return nil
-}
-
 func (h *handler) handlePprofBlock() error {
 	sec, err := strconv.ParseInt(h.rq.FormValue("seconds"), 10, 64)
 	if sec <= 0 || err != nil {

rest/api.go

@@ -22,12 +22,10 @@ import (
 	_ "net/http/pprof"
 	"net/url"
 	"os"
-	"os/signal"
 	"strconv"
 	"strings"
 	"sync"
 	"sync/atomic"
-	"syscall"
 	"time"
 
 	"github.com/go-jose/go-jose/v4"
@@ -2315,29 +2313,6 @@ func HandleSighup(ctx context.Context) {
 	base.RotateLogfiles(ctx)
 }
 
-// RegisterSignalHandler invokes functions based on the given signals:
-// - SIGHUP causes Sync Gateway to rotate log files.
-// - SIGINT or SIGTERM causes Sync Gateway to exit cleanly.
-// - SIGKILL cannot be handled by the application.
-func RegisterSignalHandler(ctx context.Context) {
-	signalChannel := make(chan os.Signal, 1)
-	signal.Notify(signalChannel, syscall.SIGHUP, os.Interrupt, syscall.SIGTERM)
-
-	go func() {
-		for sig := range signalChannel {
-			base.InfofCtx(ctx, base.KeyAll, "Handling signal: %v", sig)
-			switch sig {
-			case syscall.SIGHUP:
-				HandleSighup(ctx)
-			default:
-				// Ensure log buffers are flushed before exiting.
-				base.FlushLogBuffers()
-				os.Exit(130) // 130 == exit code 128 + 2 (interrupt)
-			}
-		}
-	}()
-}
-
 // toDbLogConfig converts the stored logging in a DbConfig to a runtime DbLogConfig for evaluation at log time.
 // This is required to turn the stored config (which does not have data stored in a O(1)-compatible format) into a data structure that has O(1) lookups for checking if we should log.
 func (c *DbConfig) toDbLogConfig(ctx context.Context) *base.DbLogConfig {

rest/config.go

@@ -16,7 +16,9 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"os/signal"
 	"path/filepath"
+	"runtime"
 	"strings"
 	"time"
 
@@ -37,7 +39,7 @@ func ServerMain() {
 
 // TODO: Pass ctx down into HTTP servers so that serverMain can be stopped.
 func serverMain(ctx context.Context, osArgs []string) error {
-	RegisterSignalHandler(ctx)
+	sigChan := RegisterSignalHandler(ctx, "")
 	defer base.FatalPanicHandler()
 
 	base.InitializeMemoryLoggers()
@@ -51,6 +53,12 @@ func serverMain(ctx context.Context, osArgs []string) error {
 		}
 		return err
 	}
+	if runtime.GOOS != "windows" {
+		// stop signal handlers and register for stack trace handling, this is not supported for windows environments
+		signal.Stop(sigChan)
+		close(sigChan)
+		RegisterSignalHandler(ctx, flagStartupConfig.Logging.LogFilePath)
+	}
 
 	if *disablePersistentConfig {
 		return legacyServerMain(ctx, osArgs, flagStartupConfig)

rest/main.go

@@ -0,0 +1,60 @@
+//go:build !windows
+// +build !windows
+
+/*
+Copyright 2025-Present Couchbase, Inc.
+
+Use of this software is governed by the Business Source License included in
+the file licenses/BSL-Couchbase.txt.  As of the Change Date specified in that
+file, in accordance with the Business Source License, use of this software will
+be governed by the Apache License, Version 2.0, included in the file
+licenses/APL2.txt.
+*/
+
+package rest
+
+import (
+	"context"
+	"os"
+	"os/signal"
+	"syscall"
+	"time"
+
+	"github.com/couchbase/sync_gateway/base"
+)
+
+// RegisterSignalHandler invokes functions based on the given signals for unix environments:
+// - SIGHUP causes Sync Gateway to rotate log files.
+// - SIGINT or SIGTERM causes Sync Gateway to exit cleanly.
+// - SIGKILL cannot be handled by the application.
+// - SIGUSR1 causes Sync Gateway to log stack traces for all goroutines.
+func RegisterSignalHandler(ctx context.Context, logDirectory string) chan os.Signal {
+	signalChannel := make(chan os.Signal, 1)
+	signal.Notify(signalChannel, syscall.SIGHUP, os.Interrupt, syscall.SIGTERM, syscall.SIGUSR1)
+
+	go func() {
+		for sig := range signalChannel {
+			base.InfofCtx(ctx, base.KeyAll, "Handling signal: %v", sig)
+			switch sig {
+			case syscall.SIGHUP:
+				HandleSighup(ctx)
+			case syscall.SIGUSR1:
+				stackTrace, err := base.GetStackTrace()
+				if err != nil {
+					base.WarnfCtx(ctx, "Error collecting stack trace: %v", err)
+				} else {
+					base.InfofCtx(ctx, base.KeyAll, "Collecting stack trace for all goroutines")
+				}
+				// log to console and log to file in the log directory
+				currentTime := time.Now()
+				timestamp := currentTime.Format(time.RFC3339)
+				base.LogStackTraces(ctx, logDirectory, stackTrace, timestamp)
+			default:
+				// Ensure log buffers are flushed before exiting.
+				base.FlushLogBuffers()
+				os.Exit(130) // 130 == exit code 128 + 2 (interrupt)
+			}
+		}
+	}()
+	return signalChannel
+}

rest/register_handler_unix.go

@@ -0,0 +1,47 @@
+//go:build windows
+// +build windows
+
+/*
+Copyright 2025-Present Couchbase, Inc.
+
+Use of this software is governed by the Business Source License included in
+the file licenses/BSL-Couchbase.txt.  As of the Change Date specified in that
+file, in accordance with the Business Source License, use of this software will
+be governed by the Apache License, Version 2.0, included in the file
+licenses/APL2.txt.
+*/
+
+package rest
+
+import (
+	"context"
+	"os"
+	"os/signal"
+	"syscall"
+
+	"github.com/couchbase/sync_gateway/base"
+)
+
+// RegisterSignalHandler invokes functions based on the given signals for windows environments:
+// - SIGHUP causes Sync Gateway to rotate log files.
+// - SIGINT or SIGTERM causes Sync Gateway to exit cleanly.
+// - SIGKILL cannot be handled by the application.
+func RegisterSignalHandler(ctx context.Context, logDirectory string) chan os.Signal {
+	signalChannel := make(chan os.Signal, 1)
+	signal.Notify(signalChannel, syscall.SIGHUP, os.Interrupt, syscall.SIGTERM)
+
+	go func() {
+		for sig := range signalChannel {
+			base.InfofCtx(ctx, base.KeyAll, "Handling signal: %v", sig)
+			switch sig {
+			case syscall.SIGHUP:
+				HandleSighup(ctx)
+			default:
+				// Ensure log buffers are flushed before exiting.
+				base.FlushLogBuffers()
+				os.Exit(130) // 130 == exit code 128 + 2 (interrupt)
+			}
+		}
+	}()
+	return signalChannel
+}