Skip to content

Commit 4d9b46e

Browse files
noahsthonoahstho
committed
profiler: add environment variable to enable continuous profiling
Adds environment variable, COCKROACH_GOOGLE_CONTINUOUS_PROFILER_ENABLED to enable continuous profiling that is "always-on" for clusters running on GCP. Variable has no effect if run on different clouds. We intend to eventually use this for getting a better picture of performance on real customer workloads as well as looking into issues on customer clusters. Short term next steps will be to verify profiler overhead is low on select internal clusters. Verified via deploying and testing on roachprod cluster, could not notice any performance degradation on 9 nodes when running TPCC benchmark on control and experiment clusters. Besides enabling this flag, setup of both the experiment and control clusters was identical, with 3 regions each running the TPCC benchmark for 2 hours. The performance of the control cluster was actually slightly worse in worst case, which statistically seemed insignificant. In addition to these experiments, after merging this change, we can proceed by testing the profiler on some internal clusters. Epic: CC-30119 Fixes: #148916 Release note: None
1 parent d0365c2 commit 4d9b46e

File tree

7 files changed

+64
-0
lines changed

7 files changed

+64
-0
lines changed

‎DEPS.bzl‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10002,6 +10002,16 @@ def go_deps():
1000210002
"https://storage.googleapis.com/cockroach-godeps/gomod/cloud.google.com/go/privatecatalog/com_google_cloud_go_privatecatalog-v0.8.0.zip",
1000310003
],
1000410004
)
10005+
go_repository(
10006+
name = "com_google_cloud_go_profiler",
10007+
build_file_proto_mode = "disable_global",
10008+
importpath = "cloud.google.com/go/profiler",
10009+
sha256 = "6026dcb7abd520ed3584a805333244b83673c28090d0e2208049178ddc757b4d",
10010+
strip_prefix = "cloud.google.com/go/[email protected]",
10011+
urls = [
10012+
"https://storage.googleapis.com/cockroach-godeps/gomod/cloud.google.com/go/profiler/com_google_cloud_go_profiler-v0.3.1.zip",
10013+
],
10014+
)
1000510015
go_repository(
1000610016
name = "com_google_cloud_go_pubsub",
1000710017
build_file_proto_mode = "disable_global",

‎build/bazelutil/distdir_files.bzl‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,7 @@ DISTDIR_FILES = {
8989
"https://storage.googleapis.com/cockroach-godeps/gomod/cloud.google.com/go/phishingprotection/com_google_cloud_go_phishingprotection-v0.7.0.zip": "7a3ce8e6b2c8f828fcd344b653849cf1e90abeca48a7eef81c75a72cb924d9e2",
9090
"https://storage.googleapis.com/cockroach-godeps/gomod/cloud.google.com/go/policytroubleshooter/com_google_cloud_go_policytroubleshooter-v1.6.0.zip": "9d5fccfe01a31ec395ba3a26474168e5a8db09275dfbdfcd5dfd44923d9ac4bd",
9191
"https://storage.googleapis.com/cockroach-godeps/gomod/cloud.google.com/go/privatecatalog/com_google_cloud_go_privatecatalog-v0.8.0.zip": "f475f487df7906e4e35bda4b69ce53f141ade7ea6463674eb9b57f5fa302c367",
92+
"https://storage.googleapis.com/cockroach-godeps/gomod/cloud.google.com/go/profiler/com_google_cloud_go_profiler-v0.3.1.zip": "6026dcb7abd520ed3584a805333244b83673c28090d0e2208049178ddc757b4d",
9293
"https://storage.googleapis.com/cockroach-godeps/gomod/cloud.google.com/go/pubsub/com_google_cloud_go_pubsub-v1.30.0.zip": "9c15c75b6204fd3d42114006896a72d82827d01a756d2f78423c101102da4977",
9394
"https://storage.googleapis.com/cockroach-godeps/gomod/cloud.google.com/go/pubsublite/com_google_cloud_go_pubsublite-v1.7.0.zip": "97b1c3637961faf18229a168a5811425b4e64ee6d81bb76e51ebbf93ff3622ba",
9495
"https://storage.googleapis.com/cockroach-godeps/gomod/cloud.google.com/go/recaptchaenterprise/v2/com_google_cloud_go_recaptchaenterprise_v2-v2.7.0.zip": "dbf218232a443651daa58869fb5e87845927c33d683f4fd4f6f4306e056bb7d0",

‎go.mod‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,7 @@ require (
8888
)
8989

9090
require (
91+
cloud.google.com/go/profiler v0.3.1
9192
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1
9293
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0
9394
github.com/Azure/azure-sdk-for-go/sdk/keyvault/azkeys v0.9.0

‎go.sum‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,8 @@ cloud.google.com/go/kms v1.10.1 h1:7hm1bRqGCA1GBRQUrp831TwJ9TWhP+tvLuP497CQS2g=
5252
cloud.google.com/go/kms v1.10.1/go.mod h1:rIWk/TryCkR59GMC3YtHtXeLzd634lBbKenvyySAyYI=
5353
cloud.google.com/go/longrunning v0.4.1 h1:v+yFJOfKC3yZdY6ZUI933pIYdhyhV8S3NpWrXWmg7jM=
5454
cloud.google.com/go/longrunning v0.4.1/go.mod h1:4iWDqhBZ70CvZ6BfETbvam3T8FMvLK+eFj0E6AaRQTo=
55+
cloud.google.com/go/profiler v0.3.1 h1:b5got9Be9Ia0HVvyt7PavWxXEht15B9lWnigdvHtxOc=
56+
cloud.google.com/go/profiler v0.3.1/go.mod h1:GsG14VnmcMFQ9b+kq71wh3EKMZr3WRMgLzNiFRpW7tE=
5557
cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
5658
cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
5759
cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=

‎pkg/cli/start.go‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ import (
3333
"github.com/cockroachdb/cockroach/pkg/security/clientsecopts"
3434
"github.com/cockroachdb/cockroach/pkg/security/username"
3535
"github.com/cockroachdb/cockroach/pkg/server"
36+
"github.com/cockroachdb/cockroach/pkg/server/profiler"
3637
"github.com/cockroachdb/cockroach/pkg/server/serverctl"
3738
"github.com/cockroachdb/cockroach/pkg/server/status"
3839
"github.com/cockroachdb/cockroach/pkg/settings"
@@ -1490,6 +1491,12 @@ func setupAndInitializeLoggingAndProfiling(
14901491
initTraceDir(ctx, serverCfg.InflightTraceDirName)
14911492
initBlockProfile()
14921493
initMutexProfile()
1494+
1495+
var gceContinuousProfilerEnabled = envutil.EnvOrDefaultBool("COCKROACH_GOOGLE_CONTINUOUS_PROFILER_ENABLED", false)
1496+
if gceContinuousProfilerEnabled {
1497+
profiler.InitGoogleProfiler(ctx)
1498+
}
1499+
14931500
log.Event(ctx, "initialized profiles")
14941501

14951502
return stopper, nil

‎pkg/server/profiler/BUILD.bazel‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ go_library(
77
"cgoprofiler.go",
88
"cluster_settings.go",
99
"cpuprofiler.go",
10+
"google_cloud_profiler.go",
1011
"heapprofiler.go",
1112
"memory_monitoring_profiler.go",
1213
"profiler_common.go",
@@ -16,13 +17,15 @@ go_library(
1617
importpath = "github.com/cockroachdb/cockroach/pkg/server/profiler",
1718
visibility = ["//visibility:public"],
1819
deps = [
20+
"//pkg/build",
1921
"//pkg/server/debug",
2022
"//pkg/server/dumpstore",
2123
"//pkg/server/status",
2224
"//pkg/settings",
2325
"//pkg/settings/cluster",
2426
"//pkg/sql",
2527
"//pkg/util/cgroups",
28+
"//pkg/util/cloudinfo",
2629
"//pkg/util/envutil",
2730
"//pkg/util/log",
2831
"//pkg/util/log/logcrash",
@@ -31,6 +34,7 @@ go_library(
3134
"//pkg/util/timeutil",
3235
"@com_github_cockroachdb_errors//:errors",
3336
"@com_github_dustin_go_humanize//:go-humanize",
37+
"@com_google_cloud_go_profiler//:profiler",
3438
],
3539
)
3640

‎pkg/server/profiler/google_cloud_profiler.go‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
// Copyright 2015 The Cockroach Authors.
2+
//
3+
// Use of this software is governed by the CockroachDB Software License
4+
// included in the /LICENSE file.
5+
6+
package profiler
7+
8+
import (
9+
"context"
10+
11+
gcprofiler "cloud.google.com/go/profiler"
12+
"github.com/cockroachdb/cockroach/pkg/build"
13+
"github.com/cockroachdb/cockroach/pkg/util/cloudinfo"
14+
"github.com/cockroachdb/cockroach/pkg/util/log"
15+
)
16+
17+
// InitGoogleProfiler initializes the Google Cloud Profiler if enabled via
18+
// environment variable, and cluster is running on GCP.
19+
//
20+
// The profiler initialization should be done as early as possible in the
21+
// server startup process for best results.
22+
func InitGoogleProfiler(ctx context.Context) {
23+
// Detect cloud provider as profiler is only supported on GCP.
24+
provider, _ := cloudinfo.GetInstanceRegion(ctx)
25+
if provider != "gcp" {
26+
log.Infof(ctx, "Google Cloud Profiler disabled (detected cloud: %s)", provider)
27+
return
28+
}
29+
30+
cfg := gcprofiler.Config{
31+
Service: "cockroachdb",
32+
ServiceVersion: build.BinaryVersion(),
33+
}
34+
if err := gcprofiler.Start(cfg); err != nil {
35+
log.Warningf(ctx, "failed to start google profiler: %v", err)
36+
} else {
37+
log.Infof(ctx, "Google Cloud Profiler started successfully on %s", provider)
38+
}
39+
}

0 commit comments

Comments
 (0)