Skip to content

Commit 85e17c6

Browse files
authored
Merge pull request #153872 from golgeek/backport25.2-141454
release-25.2: roachprod: refactor IAP authentication
2 parents 3536930 + 074db91 commit 85e17c6

File tree

14 files changed

+477
-460
lines changed

14 files changed

+477
-460
lines changed

DEPS.bzl

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1187,6 +1187,16 @@ def go_deps():
11871187
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/bgentry/speakeasy/com_github_bgentry_speakeasy-v0.1.0.zip",
11881188
],
11891189
)
1190+
go_repository(
1191+
name = "com_github_binxio_gcloudconfig",
1192+
build_file_proto_mode = "disable_global",
1193+
importpath = "github.com/binxio/gcloudconfig",
1194+
sha256 = "82797ef5d9fa4cba09d64ca885a3b6b8867d046c8f144ed15dc102085b0c6ceb",
1195+
strip_prefix = "github.com/binxio/[email protected]",
1196+
urls = [
1197+
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/binxio/gcloudconfig/com_github_binxio_gcloudconfig-v0.1.5.zip",
1198+
],
1199+
)
11901200
go_repository(
11911201
name = "com_github_biogo_store",
11921202
build_file_proto_mode = "disable_global",

build/bazelutil/distdir_files.bzl

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -297,6 +297,7 @@ DISTDIR_FILES = {
297297
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/beorn7/perks/com_github_beorn7_perks-v1.0.1.zip": "25bd9e2d94aca770e6dbc1f53725f84f6af4432f631d35dd2c46f96ef0512f1a",
298298
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/bgentry/go-netrc/com_github_bgentry_go_netrc-v0.0.0-20140422174119-9fd32a8b3d3d.zip": "59fbb1e8e307ccd7052f77186990d744284b186e8b1c5ebdfb12405ae8d7f935",
299299
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/bgentry/speakeasy/com_github_bgentry_speakeasy-v0.1.0.zip": "d4bfd48b9bf68c87f92c94478ac910bcdab272e15eb909d58f1fb939233f75f0",
300+
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/binxio/gcloudconfig/com_github_binxio_gcloudconfig-v0.1.5.zip": "82797ef5d9fa4cba09d64ca885a3b6b8867d046c8f144ed15dc102085b0c6ceb",
300301
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/biogo/store/com_github_biogo_store-v0.0.0-20160505134755-913427a1d5e8.zip": "26551f8829c5ada84a68ef240732375be6747252aba423cf5c88bc03002c3600",
301302
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/bitly/go-hostpool/com_github_bitly_go_hostpool-v0.0.0-20171023180738-a3a6125de932.zip": "9a55584d7fa2c1639d0ea11cd5b437786c2eadc2401d825e699ad6445fc8e476",
302303
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/bitly/go-simplejson/com_github_bitly_go_simplejson-v0.5.0.zip": "53930281dc7fba8947c1b1f07c82952a38dcaefae23bd3c8e71d70a6daa6cb40",

go.mod

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,7 @@ require (
120120
github.com/axiomhq/hyperloglog v0.2.5
121121
github.com/axiomhq/hyperloglog/000 v0.0.0-20181223111420-4b99d0c2c99e
122122
github.com/bazelbuild/rules_go v0.26.0
123+
github.com/binxio/gcloudconfig v0.1.5
123124
github.com/biogo/store v0.0.0-20160505134755-913427a1d5e8
124125
github.com/blevesearch/snowballstem v0.9.0
125126
github.com/buchgr/bazel-remote v1.3.3

go.sum

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -464,6 +464,8 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r
464464
github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d h1:xDfNPAt8lFiC1UJrqV3uuy861HCTo708pDMbjHHdCas=
465465
github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d/go.mod h1:6QX/PXZ00z/TKoufEY6K/a0k6AhaJrQKdFe6OfVXsa4=
466466
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
467+
github.com/binxio/gcloudconfig v0.1.5 h1:nbvWtpqn7yJs4qPuXxTu9D3DYrSyc0FHkXraseMMCV4=
468+
github.com/binxio/gcloudconfig v0.1.5/go.mod h1:IpQXzgqmv2JS1i+hbhqhHqzeYWg5zWkdN4sZJznJDUM=
467469
github.com/biogo/store v0.0.0-20160505134755-913427a1d5e8 h1:tYoz1OeRpx3dJZlh9T4dQt4kAndcmpl+VNdzbSgFC/0=
468470
github.com/biogo/store v0.0.0-20160505134755-913427a1d5e8/go.mod h1:Iev9Q3MErcn+w3UOJD/DkEzllvugfdx7bGcMOFhvr/4=
469471
github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k=

pkg/cmd/roachprod/cli/commands.go

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1664,9 +1664,9 @@ func (cr *commandRegistry) buildGrafanaAnnotationCmd() *cobra.Command {
16641664
Long: fmt.Sprintf(`Adds an annotation to the specified grafana instance
16651665
16661666
By default, we assume the grafana instance needs an authentication token to connect
1667-
to. A service account json and audience will be read in from the environment
1668-
variables %s and %s to attempt authentication through google IDP. Use the --insecure
1669-
option when a token is not necessary.
1667+
to. Unless the %s environment variable exists, the default Google Application Credentials
1668+
will be used to derive an Access Token to authenticate against Google Identity-Aware Proxy.
1669+
Use the --insecure option when a token is not necessary.
16701670
16711671
--tags specifies the tags the annotation should have.
16721672
@@ -1681,7 +1681,7 @@ creates an annotation over time range.
16811681
Example:
16821682
# Create an annotation over time range 1-100 on the centralized grafana instance, which needs authentication.
16831683
roachprod grafana-annotation grafana.testeng.crdb.io example-annotation-event --tags my-cluster --tags test-run-1 --dashboard-uid overview --time-range 1,100
1684-
`, roachprodutil.ServiceAccountJson, roachprodutil.ServiceAccountAudience),
1684+
`, roachprodutil.CredentialsEnvironmentVariable),
16851685
Args: cobra.ExactArgs(2),
16861686
Run: wrap(func(cmd *cobra.Command, args []string) error {
16871687
req := grafana.AddAnnotationRequest{

pkg/cmd/roachprod/grafana/BUILD.bazel

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,12 +22,12 @@ go_library(
2222
importpath = "github.com/cockroachdb/cockroach/pkg/cmd/roachprod/grafana",
2323
visibility = ["//visibility:public"],
2424
deps = [
25+
"//pkg/roachprod/promhelperclient",
2526
"//pkg/roachprod/roachprodutil",
2627
"//pkg/util/httputil",
2728
"@com_github_cockroachdb_errors//:errors",
2829
"@com_github_go_openapi_strfmt//:strfmt",
2930
"@com_github_grafana_grafana_openapi_client_go//client",
3031
"@com_github_grafana_grafana_openapi_client_go//models",
31-
"@org_golang_google_api//idtoken",
3232
],
3333
)

pkg/cmd/roachprod/grafana/annotations.go

Lines changed: 21 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -7,16 +7,15 @@ package grafana
77

88
import (
99
"context"
10-
"fmt"
1110
"strings"
1211

12+
"github.com/cockroachdb/cockroach/pkg/roachprod/promhelperclient"
1313
"github.com/cockroachdb/cockroach/pkg/roachprod/roachprodutil"
1414
"github.com/cockroachdb/cockroach/pkg/util/httputil"
1515
"github.com/cockroachdb/errors"
1616
"github.com/go-openapi/strfmt"
1717
grafana "github.com/grafana/grafana-openapi-client-go/client"
1818
"github.com/grafana/grafana-openapi-client-go/models"
19-
"google.golang.org/api/idtoken"
2019
)
2120

2221
// newGrafanaClient is a helper function that creates an HTTP client to
@@ -26,30 +25,37 @@ import (
2625
func newGrafanaClient(
2726
ctx context.Context, host string, secure bool,
2827
) (*grafana.GrafanaHTTPAPI, error) {
29-
headers := map[string]string{}
3028
scheme := "http"
3129

30+
// Use the default HTTP client for unsecure Grafana calls.
31+
grafanaHttpClient := httputil.DefaultClient.Client
32+
3233
if secure {
3334
scheme = "https"
3435

35-
// Read in the service account key and audience, so we can retrieve the identity token.
36-
if _, err := roachprodutil.SetServiceAccountCredsEnv(ctx, false); err != nil {
37-
return nil, err
38-
}
39-
40-
token, err := roachprodutil.GetServiceAccountToken(ctx, idtoken.NewTokenSource)
36+
// Grafana annotations currently use the same service account
37+
// and OAuth client ID as the prometheus helper service.
38+
iapTokenSource, err := roachprodutil.NewIAPTokenSource(roachprodutil.IAPTokenSourceOptions{
39+
OAuthClientID: promhelperclient.OAuthClientID,
40+
ServiceAccountEmail: promhelperclient.ServiceAccountEmail,
41+
})
4142
if err != nil {
4243
return nil, err
4344
}
44-
headers["Authorization"] = fmt.Sprintf("Bearer %s", token)
45+
46+
// Override the default HTTP client with the one
47+
// that has the IAP token source.
48+
grafanaHttpClient = iapTokenSource.GetHTTPClient()
4549
}
4650

47-
headers[httputil.ContentTypeHeader] = httputil.JSONContentType
4851
cfg := &grafana.TransportConfig{
49-
Host: host,
50-
BasePath: "/api",
51-
Schemes: []string{scheme},
52-
HTTPHeaders: headers,
52+
Host: host,
53+
BasePath: "/api",
54+
Schemes: []string{scheme},
55+
HTTPHeaders: map[string]string{
56+
httputil.ContentTypeHeader: httputil.JSONContentType,
57+
},
58+
Client: grafanaHttpClient,
5359
}
5460

5561
return grafana.NewHTTPClientWithConfig(strfmt.Default, cfg), nil

pkg/roachprod/cloud/cluster_cloud.go

Lines changed: 16 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -487,11 +487,14 @@ func ShrinkCluster(l *logger.Logger, c *Cluster, numNodes int) error {
487487

488488
func (c *Cluster) DeletePrometheusConfig(ctx context.Context, l *logger.Logger) error {
489489

490-
cl := promhelperclient.NewPromClient()
491-
492490
stopSpinner := ui.NewDefaultSpinner(l, "Destroying Prometheus configs").Start()
493491
defer stopSpinner()
494492

493+
// We first iterate on all VMs to determine if any machine of the cluster
494+
// was reachable by Prometheus and if we need to delete its config.
495+
// This is done this way to avoid authenticating the promhelper client
496+
// in case we don't need to delete any config.
497+
needDelete := false
495498
for _, node := range c.VMs {
496499

497500
reachability := promhelperclient.ProviderReachability(
@@ -502,29 +505,20 @@ func (c *Cluster) DeletePrometheusConfig(ctx context.Context, l *logger.Logger)
502505
continue
503506
}
504507

505-
err := cl.DeleteClusterConfig(ctx, c.Name, false, false /* insecure */, l)
506-
if err != nil {
507-
508-
if !promhelperclient.IsNotFoundError(err) {
509-
return errors.Wrapf(
510-
err,
511-
"failed to delete the cluster config with cluster as secure",
512-
)
513-
}
514-
515-
// TODO(bhaskar): Obtain secure cluster information.
516-
// Cluster does not have the information on secure or not.
517-
// So, we retry as insecure if delete fails with cluster as secure.
518-
if err = cl.DeleteClusterConfig(ctx, c.Name, false, true /* insecure */, l); err != nil {
519-
return errors.Wrapf(
520-
err,
521-
"failed to delete the cluster config with cluster as insecure and secure",
522-
)
523-
}
508+
needDelete = true
509+
break
510+
}
524511

512+
if needDelete {
513+
cl, err := promhelperclient.NewPromClient()
514+
if err != nil {
515+
return err
525516
}
526-
break
527517

518+
err = cl.DeleteClusterConfig(ctx, c.Name, l)
519+
if err != nil {
520+
return err
521+
}
528522
}
529523

530524
return nil

pkg/roachprod/promhelperclient/BUILD.bazel

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,6 @@ go_library(
1515
"//pkg/util/httputil",
1616
"@com_github_cockroachdb_errors//:errors",
1717
"@in_gopkg_yaml_v2//:yaml_v2",
18-
"@org_golang_google_api//idtoken",
19-
"@org_golang_x_oauth2//:oauth2",
2018
],
2119
)
2220

@@ -26,10 +24,8 @@ go_test(
2624
embed = [":promhelperclient"],
2725
deps = [
2826
"//pkg/roachprod/logger",
29-
"//pkg/roachprod/roachprodutil",
3027
"@com_github_stretchr_testify//require",
3128
"@in_gopkg_yaml_v2//:yaml_v2",
32-
"@org_golang_google_api//idtoken",
3329
"@org_golang_x_oauth2//:oauth2",
3430
],
3531
)

0 commit comments

Comments
 (0)