Skip to content

Commit d811a31

Browse files
craig[bot]noahstho
craig[bot]
and
noahstho
committed
Merge #149242
149242: profiler: add environment variable to enable continuous profiling r=noahstho a=noahstho Adds environment variable, COCKROACH_GOOGLE_CONTINUOUS_PROFILER_ENABLED to enable continuous profiling that is "always-on" for clusters running on GCP. Variable has no effect if run on different clouds. We intend to eventually use this for getting a better picture of performance on real customer workloads as well as looking into issues on customer clusters. Short term next steps will be to verify profiler overhead is low on select internal clusters. Verified via deploying and testing on roachprod cluster, could not notice any performance degradation on 10 nodes when running TPCC benchmark on control and experiment clusters. Epic: CC-30119 Fixes: #148916 Release note: None Co-authored-by: Noah Thompson <[email protected]>
2 parents d591c19 + 4d9b46e commit d811a31

File tree

7 files changed

+64
-0
lines changed

7 files changed

+64
-0
lines changed

‎DEPS.bzl‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10002,6 +10002,16 @@ def go_deps():
1000210002
"https://storage.googleapis.com/cockroach-godeps/gomod/cloud.google.com/go/privatecatalog/com_google_cloud_go_privatecatalog-v0.8.0.zip",
1000310003
],
1000410004
)
10005+
go_repository(
10006+
name = "com_google_cloud_go_profiler",
10007+
build_file_proto_mode = "disable_global",
10008+
importpath = "cloud.google.com/go/profiler",
10009+
sha256 = "6026dcb7abd520ed3584a805333244b83673c28090d0e2208049178ddc757b4d",
10010+
strip_prefix = "cloud.google.com/go/[email protected]",
10011+
urls = [
10012+
"https://storage.googleapis.com/cockroach-godeps/gomod/cloud.google.com/go/profiler/com_google_cloud_go_profiler-v0.3.1.zip",
10013+
],
10014+
)
1000510015
go_repository(
1000610016
name = "com_google_cloud_go_pubsub",
1000710017
build_file_proto_mode = "disable_global",

‎build/bazelutil/distdir_files.bzl‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,7 @@ DISTDIR_FILES = {
8989
"https://storage.googleapis.com/cockroach-godeps/gomod/cloud.google.com/go/phishingprotection/com_google_cloud_go_phishingprotection-v0.7.0.zip": "7a3ce8e6b2c8f828fcd344b653849cf1e90abeca48a7eef81c75a72cb924d9e2",
9090
"https://storage.googleapis.com/cockroach-godeps/gomod/cloud.google.com/go/policytroubleshooter/com_google_cloud_go_policytroubleshooter-v1.6.0.zip": "9d5fccfe01a31ec395ba3a26474168e5a8db09275dfbdfcd5dfd44923d9ac4bd",
9191
"https://storage.googleapis.com/cockroach-godeps/gomod/cloud.google.com/go/privatecatalog/com_google_cloud_go_privatecatalog-v0.8.0.zip": "f475f487df7906e4e35bda4b69ce53f141ade7ea6463674eb9b57f5fa302c367",
92+
"https://storage.googleapis.com/cockroach-godeps/gomod/cloud.google.com/go/profiler/com_google_cloud_go_profiler-v0.3.1.zip": "6026dcb7abd520ed3584a805333244b83673c28090d0e2208049178ddc757b4d",
9293
"https://storage.googleapis.com/cockroach-godeps/gomod/cloud.google.com/go/pubsub/com_google_cloud_go_pubsub-v1.30.0.zip": "9c15c75b6204fd3d42114006896a72d82827d01a756d2f78423c101102da4977",
9394
"https://storage.googleapis.com/cockroach-godeps/gomod/cloud.google.com/go/pubsublite/com_google_cloud_go_pubsublite-v1.7.0.zip": "97b1c3637961faf18229a168a5811425b4e64ee6d81bb76e51ebbf93ff3622ba",
9495
"https://storage.googleapis.com/cockroach-godeps/gomod/cloud.google.com/go/recaptchaenterprise/v2/com_google_cloud_go_recaptchaenterprise_v2-v2.7.0.zip": "dbf218232a443651daa58869fb5e87845927c33d683f4fd4f6f4306e056bb7d0",

‎go.mod‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,7 @@ require (
8888
)
8989

9090
require (
91+
cloud.google.com/go/profiler v0.3.1
9192
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1
9293
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0
9394
github.com/Azure/azure-sdk-for-go/sdk/keyvault/azkeys v0.9.0

‎go.sum‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,8 @@ cloud.google.com/go/kms v1.10.1 h1:7hm1bRqGCA1GBRQUrp831TwJ9TWhP+tvLuP497CQS2g=
5252
cloud.google.com/go/kms v1.10.1/go.mod h1:rIWk/TryCkR59GMC3YtHtXeLzd634lBbKenvyySAyYI=
5353
cloud.google.com/go/longrunning v0.4.1 h1:v+yFJOfKC3yZdY6ZUI933pIYdhyhV8S3NpWrXWmg7jM=
5454
cloud.google.com/go/longrunning v0.4.1/go.mod h1:4iWDqhBZ70CvZ6BfETbvam3T8FMvLK+eFj0E6AaRQTo=
55+
cloud.google.com/go/profiler v0.3.1 h1:b5got9Be9Ia0HVvyt7PavWxXEht15B9lWnigdvHtxOc=
56+
cloud.google.com/go/profiler v0.3.1/go.mod h1:GsG14VnmcMFQ9b+kq71wh3EKMZr3WRMgLzNiFRpW7tE=
5557
cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
5658
cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
5759
cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=

‎pkg/cli/start.go‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ import (
3333
"github.com/cockroachdb/cockroach/pkg/security/clientsecopts"
3434
"github.com/cockroachdb/cockroach/pkg/security/username"
3535
"github.com/cockroachdb/cockroach/pkg/server"
36+
"github.com/cockroachdb/cockroach/pkg/server/profiler"
3637
"github.com/cockroachdb/cockroach/pkg/server/serverctl"
3738
"github.com/cockroachdb/cockroach/pkg/server/status"
3839
"github.com/cockroachdb/cockroach/pkg/settings"
@@ -1490,6 +1491,12 @@ func setupAndInitializeLoggingAndProfiling(
14901491
initTraceDir(ctx, serverCfg.InflightTraceDirName)
14911492
initBlockProfile()
14921493
initMutexProfile()
1494+
1495+
var gceContinuousProfilerEnabled = envutil.EnvOrDefaultBool("COCKROACH_GOOGLE_CONTINUOUS_PROFILER_ENABLED", false)
1496+
if gceContinuousProfilerEnabled {
1497+
profiler.InitGoogleProfiler(ctx)
1498+
}
1499+
14931500
log.Event(ctx, "initialized profiles")
14941501

14951502
return stopper, nil

‎pkg/server/profiler/BUILD.bazel‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ go_library(
77
"cgoprofiler.go",
88
"cluster_settings.go",
99
"cpuprofiler.go",
10+
"google_cloud_profiler.go",
1011
"heapprofiler.go",
1112
"memory_monitoring_profiler.go",
1213
"profiler_common.go",
@@ -16,13 +17,15 @@ go_library(
1617
importpath = "github.com/cockroachdb/cockroach/pkg/server/profiler",
1718
visibility = ["//visibility:public"],
1819
deps = [
20+
"//pkg/build",
1921
"//pkg/server/debug",
2022
"//pkg/server/dumpstore",
2123
"//pkg/server/status",
2224
"//pkg/settings",
2325
"//pkg/settings/cluster",
2426
"//pkg/sql",
2527
"//pkg/util/cgroups",
28+
"//pkg/util/cloudinfo",
2629
"//pkg/util/envutil",
2730
"//pkg/util/log",
2831
"//pkg/util/log/logcrash",
@@ -31,6 +34,7 @@ go_library(
3134
"//pkg/util/timeutil",
3235
"@com_github_cockroachdb_errors//:errors",
3336
"@com_github_dustin_go_humanize//:go-humanize",
37+
"@com_google_cloud_go_profiler//:profiler",
3438
],
3539
)
3640

‎pkg/server/profiler/google_cloud_profiler.go‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
// Copyright 2015 The Cockroach Authors.
2+
//
3+
// Use of this software is governed by the CockroachDB Software License
4+
// included in the /LICENSE file.
5+
6+
package profiler
7+
8+
import (
9+
"context"
10+
11+
gcprofiler "cloud.google.com/go/profiler"
12+
"github.com/cockroachdb/cockroach/pkg/build"
13+
"github.com/cockroachdb/cockroach/pkg/util/cloudinfo"
14+
"github.com/cockroachdb/cockroach/pkg/util/log"
15+
)
16+
17+
// InitGoogleProfiler initializes the Google Cloud Profiler if enabled via
18+
// environment variable, and cluster is running on GCP.
19+
//
20+
// The profiler initialization should be done as early as possible in the
21+
// server startup process for best results.
22+
func InitGoogleProfiler(ctx context.Context) {
23+
// Detect cloud provider as profiler is only supported on GCP.
24+
provider, _ := cloudinfo.GetInstanceRegion(ctx)
25+
if provider != "gcp" {
26+
log.Infof(ctx, "Google Cloud Profiler disabled (detected cloud: %s)", provider)
27+
return
28+
}
29+
30+
cfg := gcprofiler.Config{
31+
Service: "cockroachdb",
32+
ServiceVersion: build.BinaryVersion(),
33+
}
34+
if err := gcprofiler.Start(cfg); err != nil {
35+
log.Warningf(ctx, "failed to start google profiler: %v", err)
36+
} else {
37+
log.Infof(ctx, "Google Cloud Profiler started successfully on %s", provider)
38+
}
39+
}

0 commit comments

Comments
 (0)