Skip to content

Commit e54a721

Browse files
committed
updates based off review
1 parent 8207199 commit e54a721

16 files changed

+164
-203
lines changed

base/constants.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -155,6 +155,9 @@ const (
155155

156156
// FromConnStrWarningThreshold determines the amount of time it should take before we warn about parsing a connstr (mostly for DNS resolution)
157157
FromConnStrWarningThreshold = 10 * time.Second
158+
159+
// StackFilePrefix is the prefix used when writing stack trace files
160+
StackFilePrefix = "sg_stack_trace_"
158161
)
159162

160163
// SyncGatewayRawDocXattrs is a list of xattrs that Sync Gateway will fetch when reading a raw document.

base/util.go

Lines changed: 34 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ import (
3333
"regexp"
3434
"runtime"
3535
"runtime/debug"
36+
"runtime/pprof"
3637
"slices"
3738
"sort"
3839
"strconv"
@@ -1835,19 +1836,10 @@ func IsRevTreeID(s string) bool {
18351836
}
18361837

18371838
// GetStackTrace will return goroutine stack traces for all goroutines in Sync Gateway.
1838-
func GetStackTrace() string {
1839-
// make 1MB buffer but if this buffer isn't big enough, runtime.Stack will
1840-
// return nothing, thus have 5 retires doubling the capacity each time.
1841-
buf := make([]byte, 1<<20)
1842-
for range 5 {
1843-
n := runtime.Stack(buf, true)
1844-
if n < len(buf) {
1845-
buf = buf[:n]
1846-
break
1847-
}
1848-
buf = make([]byte, 2*len(buf))
1849-
}
1850-
return string(buf)
1839+
func GetStackTrace() (bytes.Buffer, error) {
1840+
profBuf := bytes.Buffer{}
1841+
err := pprof.Lookup("goroutine").WriteTo(&profBuf, 2)
1842+
return profBuf, err
18511843
}
18521844

18531845
// RotateProfilesIfNeeded will remove old files if there are more than
@@ -1870,3 +1862,32 @@ func RotateProfilesIfNeeded(filename string) error {
18701862
}
18711863
return multiErr.ErrorOrNil()
18721864
}
1865+
1866+
func LogStackTraces(ctx context.Context, logDirectory string, stackTrace bytes.Buffer, timestamp string) {
1867+
1868+
// log to console
1869+
_, _ = fmt.Fprintf(os.Stderr, "Stack trace:\n%s\n", stackTrace.String())
1870+
1871+
filename := filepath.Join(logDirectory, StackFilePrefix+timestamp+".log")
1872+
file, err := os.Create(filename)
1873+
defer func() {
1874+
closeErr := file.Close()
1875+
if closeErr != nil {
1876+
WarnfCtx(ctx, "Error closing stack trace file %s: %v", filename, closeErr)
1877+
}
1878+
}()
1879+
if err != nil {
1880+
WarnfCtx(ctx, "Error opening stack trace file %s: %v", filename, err)
1881+
}
1882+
1883+
_, err = file.WriteString(fmt.Sprintf("Stack trace:\n%s\n", stackTrace.String()))
1884+
if err != nil {
1885+
WarnfCtx(ctx, "Error writing stack trace to file %s: %v", filename, err)
1886+
}
1887+
1888+
rotatePath := filepath.Join(logDirectory, StackFilePrefix+"*.log")
1889+
err = RotateProfilesIfNeeded(rotatePath)
1890+
if err != nil {
1891+
WarnfCtx(ctx, "Error rotating stack trace files in path %s: %v", rotatePath, err)
1892+
}
1893+
}

docs/api/admin.yaml

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -97,8 +97,6 @@ paths:
9797
$ref: ./paths/admin/_debug-pprof-trace.yaml
9898
/_debug/fgprof:
9999
$ref: ./paths/admin/_debug-fgprof.yaml
100-
/_debug/stacktrace:
101-
$ref: ./paths/admin/_debug-stack_trace.yaml
102100
/_post_upgrade:
103101
$ref: ./paths/admin/_post_upgrade.yaml
104102
'/{db}/_config':

docs/api/paths/admin/_debug-stack_trace.yaml

Lines changed: 0 additions & 25 deletions
This file was deleted.

rest/adminapitest/admin_api_test.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2273,11 +2273,11 @@ func TestHandleGetStackTrace(t *testing.T) {
22732273
rt := rest.NewRestTester(t, nil)
22742274
defer rt.Close()
22752275

2276-
resp := rt.SendAdminRequest(http.MethodGet, "/_debug/stacktrace", "")
2276+
resp := rt.SendAdminRequest(http.MethodGet, "/_debug/pprof/goroutine?debug=2", "")
22772277
rest.RequireStatus(t, resp, http.StatusOK)
22782278
rawResponseStr := resp.Body.String()
22792279
assert.Contains(t, rawResponseStr, "goroutine")
2280-
assert.Contains(t, rawResponseStr, "handleCollectStackTrace")
2280+
assert.Contains(t, rawResponseStr, "handlePprofGoroutine")
22812281
}
22822282

22832283
func TestConfigRedaction(t *testing.T) {

rest/api.go

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -693,14 +693,6 @@ func (h *handler) handleFgprof() error {
693693
return stopFn()
694694
}
695695

696-
func (h *handler) handleCollectStackTrace() error {
697-
698-
stackTrace := base.GetStackTrace()
699-
700-
h.writeText([]byte(stackTrace))
701-
return nil
702-
}
703-
704696
func (h *handler) handlePprofBlock() error {
705697
sec, err := strconv.ParseInt(h.rq.FormValue("seconds"), 10, 64)
706698
if sec <= 0 || err != nil {

rest/config.go

Lines changed: 0 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -22,12 +22,10 @@ import (
2222
_ "net/http/pprof"
2323
"net/url"
2424
"os"
25-
"os/signal"
2625
"strconv"
2726
"strings"
2827
"sync"
2928
"sync/atomic"
30-
"syscall"
3129
"time"
3230

3331
"github.com/go-jose/go-jose/v4"
@@ -2315,29 +2313,6 @@ func HandleSighup(ctx context.Context) {
23152313
base.RotateLogfiles(ctx)
23162314
}
23172315

2318-
// RegisterSignalHandler invokes functions based on the given signals:
2319-
// - SIGHUP causes Sync Gateway to rotate log files.
2320-
// - SIGINT or SIGTERM causes Sync Gateway to exit cleanly.
2321-
// - SIGKILL cannot be handled by the application.
2322-
func RegisterSignalHandler(ctx context.Context) {
2323-
signalChannel := make(chan os.Signal, 1)
2324-
signal.Notify(signalChannel, syscall.SIGHUP, os.Interrupt, syscall.SIGTERM)
2325-
2326-
go func() {
2327-
for sig := range signalChannel {
2328-
base.InfofCtx(ctx, base.KeyAll, "Handling signal: %v", sig)
2329-
switch sig {
2330-
case syscall.SIGHUP:
2331-
HandleSighup(ctx)
2332-
default:
2333-
// Ensure log buffers are flushed before exiting.
2334-
base.FlushLogBuffers()
2335-
os.Exit(130) // 130 == exit code 128 + 2 (interrupt)
2336-
}
2337-
}
2338-
}()
2339-
}
2340-
23412316
// toDbLogConfig converts the stored logging in a DbConfig to a runtime DbLogConfig for evaluation at log time.
23422317
// This is required to turn the stored config (which does not have data stored in a O(1)-compatible format) into a data structure that has O(1) lookups for checking if we should log.
23432318
func (c *DbConfig) toDbLogConfig(ctx context.Context) *base.DbLogConfig {

rest/main.go

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,9 @@ import (
1616
"fmt"
1717
"io"
1818
"os"
19+
"os/signal"
1920
"path/filepath"
21+
"runtime"
2022
"strings"
2123
"time"
2224

@@ -37,7 +39,7 @@ func ServerMain() {
3739

3840
// TODO: Pass ctx down into HTTP servers so that serverMain can be stopped.
3941
func serverMain(ctx context.Context, osArgs []string) error {
40-
RegisterSignalHandler(ctx)
42+
sigChan := RegisterSignalHandler(ctx, "")
4143
defer base.FatalPanicHandler()
4244

4345
base.InitializeMemoryLoggers()
@@ -51,6 +53,12 @@ func serverMain(ctx context.Context, osArgs []string) error {
5153
}
5254
return err
5355
}
56+
if runtime.GOOS != "windows" {
57+
// stop signal handlers and register for stack trace handling, this is not supported for windows environments
58+
signal.Stop(sigChan)
59+
close(sigChan)
60+
RegisterSignalHandler(ctx, flagStartupConfig.Logging.LogFilePath)
61+
}
5462

5563
if *disablePersistentConfig {
5664
return legacyServerMain(ctx, osArgs, flagStartupConfig)

rest/register_handler_unix.go

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
//go:build !windows
2+
// +build !windows
3+
4+
/*
5+
Copyright 2025-Present Couchbase, Inc.
6+
7+
Use of this software is governed by the Business Source License included in
8+
the file licenses/BSL-Couchbase.txt. As of the Change Date specified in that
9+
file, in accordance with the Business Source License, use of this software will
10+
be governed by the Apache License, Version 2.0, included in the file
11+
licenses/APL2.txt.
12+
*/
13+
14+
package rest
15+
16+
import (
17+
"context"
18+
"os"
19+
"os/signal"
20+
"syscall"
21+
"time"
22+
23+
"github.com/couchbase/sync_gateway/base"
24+
)
25+
26+
// RegisterSignalHandler invokes functions based on the given signals for unix environments:
27+
// - SIGHUP causes Sync Gateway to rotate log files.
28+
// - SIGINT or SIGTERM causes Sync Gateway to exit cleanly.
29+
// - SIGKILL cannot be handled by the application.
30+
// - SIGUSR1 causes Sync Gateway to log stack traces for all goroutines.
31+
func RegisterSignalHandler(ctx context.Context, logDirectory string) chan os.Signal {
32+
signalChannel := make(chan os.Signal, 1)
33+
signal.Notify(signalChannel, syscall.SIGHUP, os.Interrupt, syscall.SIGTERM, syscall.SIGUSR1)
34+
35+
go func() {
36+
for sig := range signalChannel {
37+
base.InfofCtx(ctx, base.KeyAll, "Handling signal: %v", sig)
38+
switch sig {
39+
case syscall.SIGHUP:
40+
HandleSighup(ctx)
41+
case syscall.SIGUSR1:
42+
stackTrace, err := base.GetStackTrace()
43+
if err != nil {
44+
base.WarnfCtx(ctx, "Error collecting stack trace: %v", err)
45+
} else {
46+
base.InfofCtx(ctx, base.KeyAll, "Collecting stack trace for all goroutines")
47+
}
48+
// log to console and log to file in the log directory
49+
currentTime := time.Now()
50+
timestamp := currentTime.Format(time.RFC3339)
51+
base.LogStackTraces(ctx, logDirectory, stackTrace, timestamp)
52+
default:
53+
// Ensure log buffers are flushed before exiting.
54+
base.FlushLogBuffers()
55+
os.Exit(130) // 130 == exit code 128 + 2 (interrupt)
56+
}
57+
}
58+
}()
59+
return signalChannel
60+
}

rest/register_handler_windows.go

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
//go:build windows
2+
// +build windows
3+
4+
/*
5+
Copyright 2025-Present Couchbase, Inc.
6+
7+
Use of this software is governed by the Business Source License included in
8+
the file licenses/BSL-Couchbase.txt. As of the Change Date specified in that
9+
file, in accordance with the Business Source License, use of this software will
10+
be governed by the Apache License, Version 2.0, included in the file
11+
licenses/APL2.txt.
12+
*/
13+
14+
package rest
15+
16+
import (
17+
"context"
18+
"os"
19+
"os/signal"
20+
"syscall"
21+
22+
"github.com/couchbase/sync_gateway/base"
23+
)
24+
25+
// RegisterSignalHandler invokes functions based on the given signals for windows environments:
26+
// - SIGHUP causes Sync Gateway to rotate log files.
27+
// - SIGINT or SIGTERM causes Sync Gateway to exit cleanly.
28+
// - SIGKILL cannot be handled by the application.
29+
func RegisterSignalHandler(ctx context.Context, logDirectory string) chan os.Signal {
30+
signalChannel := make(chan os.Signal, 1)
31+
signal.Notify(signalChannel, syscall.SIGHUP, os.Interrupt, syscall.SIGTERM)
32+
33+
go func() {
34+
for sig := range signalChannel {
35+
base.InfofCtx(ctx, base.KeyAll, "Handling signal: %v", sig)
36+
switch sig {
37+
case syscall.SIGHUP:
38+
HandleSighup(ctx)
39+
default:
40+
// Ensure log buffers are flushed before exiting.
41+
base.FlushLogBuffers()
42+
os.Exit(130) // 130 == exit code 128 + 2 (interrupt)
43+
}
44+
}
45+
}()
46+
return signalChannel
47+
}

0 commit comments

Comments
 (0)