Merge #143633

143633: server: add EngineStats endpoint in multitenant setup r=shubhamdhama,cthumuluru-crdb a=shaikzakiriitm

EngineStats endpoint was inaccessible from secondary tenant.

EngineStats endpoint provides statistics of storage layer which can help diagnose myriad of issues critical to database performance.
To address this, we created EngineStatus endpoint in tenant status server. Implementation uses tenant connector to redirect call to system status server on requested kv node. Access to this endpoint is guarded by `can_view_node_info` capability as this endpoint returns info of all stores on a given node. Updated corresponding unit tests.

Epic: CRDB-38968
Fixes: #110020

Release note: None

Co-authored-by: Shaik Zakir Hussain <[email protected]>

Author: craig[bot]

pkg/ccl/changefeedccl/mocks/tenant_status_server_generated.go

@@ -43,9 +43,7 @@ debug zip --concurrency=1 --cpu-profile-duration=1s /dev/null
 [node 1] requesting stacks... received response... writing binary output: debug/nodes/1/stacks.txt... done
 [node 1] requesting stacks with labels... received response... writing binary output: debug/nodes/1/stacks_with_labels.txt... done
 [node 1] requesting heap profile... received response... writing binary output: debug/nodes/1/heap.pprof... done
-[node 1] requesting engine stats... received response...
-[node 1] requesting engine stats: last request failed: rpc error: ...
-[node 1] requesting engine stats: creating error output: debug/nodes/1/lsm.txt.err.txt... done
+[node 1] requesting engine stats... received response... writing binary output: debug/nodes/1/lsm.txt... done
 [node 1] requesting heap profile list... received response... done
 [node ?] ? heap profiles found
 [node 1] requesting goroutine dump list... received response... done

pkg/cli/testdata/zip/testzip_external_process_virtualization

@@ -82,9 +82,7 @@ debug zip --concurrency=1 --cpu-profile-duration=1s /dev/null
 [node 1] requesting stacks... received response... writing binary output: debug/cluster/test-tenant/nodes/1/stacks.txt... done
 [node 1] requesting stacks with labels... received response... writing binary output: debug/cluster/test-tenant/nodes/1/stacks_with_labels.txt... done
 [node 1] requesting heap profile... received response... writing binary output: debug/cluster/test-tenant/nodes/1/heap.pprof... done
-[node 1] requesting engine stats... received response...
-[node 1] requesting engine stats: last request failed: rpc error: ...
-[node 1] requesting engine stats: creating error output: debug/cluster/test-tenant/nodes/1/lsm.txt.err.txt... done
+[node 1] requesting engine stats... received response... writing binary output: debug/cluster/test-tenant/nodes/1/lsm.txt... done
 [node 1] requesting heap profile list... received response...
 [node 1] requesting heap profile list: last request failed: rpc error: ...
 [node 1] requesting heap profile list: creating error output: debug/cluster/test-tenant/nodes/1/heapprof.err.txt... done

pkg/cli/testdata/zip/testzip_shared_process_virtualization

@@ -82,9 +82,7 @@ debug zip --concurrency=1 --cpu-profile-duration=1s /dev/null
 [node 1] requesting stacks... received response... writing binary output: debug/cluster/test-tenant/nodes/1/stacks.txt... done
 [node 1] requesting stacks with labels... received response... writing binary output: debug/cluster/test-tenant/nodes/1/stacks_with_labels.txt... done
 [node 1] requesting heap profile... received response... writing binary output: debug/cluster/test-tenant/nodes/1/heap.pprof... done
-[node 1] requesting engine stats... received response...
-[node 1] requesting engine stats: last request failed: rpc error: ...
-[node 1] requesting engine stats: creating error output: debug/cluster/test-tenant/nodes/1/lsm.txt.err.txt... done
+[node 1] requesting engine stats... received response... writing binary output: debug/cluster/test-tenant/nodes/1/lsm.txt... done
 [node 1] requesting heap profile list... received response...
 [node 1] requesting heap profile list: last request failed: rpc error: ...
 [node 1] requesting heap profile list: creating error output: debug/cluster/test-tenant/nodes/1/heapprof.err.txt... done

pkg/cli/testdata/zip/testzip_shared_process_virtualization_with_default_tenant

@@ -635,6 +635,16 @@ func (c *connector) Gossip(
 	return
 }
 
+func (c *connector) EngineStats(
+	ctx context.Context, req *serverpb.EngineStatsRequest,
+) (resp *serverpb.EngineStatsResponse, retErr error) {
+	retErr = c.withClient(ctx, func(ctx context.Context, client *client) (err error) {
+		resp, err = client.EngineStats(ctx, req)
+		return
+	})
+	return
+}
+
 // NewIterator implements the rangedesc.IteratorFactory interface.
 func (c *connector) NewIterator(
 	ctx context.Context, span roachpb.Span,

pkg/kv/kvclient/kvtenant/connector.go

@@ -121,7 +121,7 @@ func (a tenantAuthorizer) authorize(
 	case "/cockroach.server.serverpb.Status/NetworkConnectivity":
 		return a.capabilitiesAuthorizer.HasProcessDebugCapability(ctx, tenID)
 
-	case "/cockroach.server.serverpb.Status/Gossip":
+	case "/cockroach.server.serverpb.Status/Gossip", "/cockroach.server.serverpb.Status/EngineStats":
 		return a.capabilitiesAuthorizer.HasNodeStatusCapability(ctx, tenID)
 
 	case "/cockroach.server.serverpb.Status/TransactionContentionEvents":

pkg/rpc/auth_tenant.go

@@ -94,6 +94,7 @@ type TenantStatusServer interface {
 	DownloadSpan(ctx context.Context, request *DownloadSpanRequest) (*DownloadSpanResponse, error)
 	NetworkConnectivity(context.Context, *NetworkConnectivityRequest) (*NetworkConnectivityResponse, error)
 	Gossip(context.Context, *GossipRequest) (*gossip.InfoStatus, error)
+	EngineStats(context.Context, *EngineStatsRequest) (*EngineStatsResponse, error)
 }
 
 // OptionalNodesStatusServer returns the wrapped NodesStatusServer, if it is

pkg/server/serverpb/status.go

@@ -788,6 +788,22 @@ func (s *statusServer) redactGossipResponse(resp *gossip.InfoStatus) *gossip.Inf
 	return resp
 }
 
+// EngineStats returns statistical information of storage layer on the given node
+// which is crucial for diagnosing issues related to disk usage,compaction efficiency,
+// read/write amplification and other storage engine metrics critical for database
+// performance.
+func (t *statusServer) EngineStats(
+	ctx context.Context, req *serverpb.EngineStatsRequest,
+) (*serverpb.EngineStatsResponse, error) {
+	ctx = t.AnnotateCtx(ctx)
+
+	if err := t.privilegeChecker.RequireViewClusterMetadataPermission(ctx); err != nil {
+		return nil, err
+	}
+
+	return t.sqlServer.tenantConnect.EngineStats(ctx, req)
+}
+
 func (s *systemStatusServer) EngineStats(
 	ctx context.Context, req *serverpb.EngineStatsRequest,
 ) (*serverpb.EngineStatsResponse, error) {

pkg/server/status.go

@@ -8,9 +8,11 @@ package storage_api_test
 import (
 	"context"
 	"regexp"
+	"strings"
 	"testing"
 
 	"github.com/cockroachdb/cockroach/pkg/base"
+	"github.com/cockroachdb/cockroach/pkg/multitenant/tenantcapabilitiespb"
 	"github.com/cockroachdb/cockroach/pkg/server/debug"
 	"github.com/cockroachdb/cockroach/pkg/server/serverpb"
 	"github.com/cockroachdb/cockroach/pkg/server/srvtestutils"
@@ -19,25 +21,34 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/util/leaktest"
 	"github.com/cockroachdb/cockroach/pkg/util/log"
 	"github.com/pkg/errors"
+	"github.com/stretchr/testify/require"
 )
 
 // TestStatusEngineStatsJson ensures that the output response for the engine
 // stats contains the required fields.
 func TestStatusEngineStatsJson(t *testing.T) {
 	defer leaktest.AfterTest(t)()
 	defer log.Scope(t).Close(t)
+	ctx := context.Background()
 
 	dir, cleanupFn := testutils.TempDir(t)
 	defer cleanupFn()
 
 	srv := serverutils.StartServerOnly(t, base.TestServerArgs{
-		DefaultTestTenant: base.TestIsForStuffThatShouldWorkWithSecondaryTenantsButDoesntYet(110020),
-
 		StoreSpecs: []base.StoreSpec{{
 			Path: dir,
 		}},
 	})
-	defer srv.Stopper().Stop(context.Background())
+	defer srv.Stopper().Stop(ctx)
+
+	if srv.DeploymentMode().IsExternal() {
+		// Explicitly enabling CanViewNodeInfo capability for the secondary/application tenant
+		// when in external process mode, as shared process mode already has all capabilities.
+		require.NoError(t, srv.GrantTenantCapabilities(
+			ctx, serverutils.TestTenantID(),
+			map[tenantcapabilitiespb.ID]string{tenantcapabilitiespb.CanViewNodeInfo: "true"}))
+	}
+
 	s := srv.ApplicationLayer()
 
 	t.Logf("using admin URL %s", s.AdminURL())
@@ -57,3 +68,32 @@ func TestStatusEngineStatsJson(t *testing.T) {
 		t.Fatal(errors.Errorf("expected engine metrics to be correctly formatted, got:\n %s", formattedStats))
 	}
 }
+
+func TestStatusEngineStatsJsonWithoutTenantCapability(t *testing.T) {
+	defer leaktest.AfterTest(t)()
+	defer log.Scope(t).Close(t)
+
+	srv := serverutils.StartServerOnly(t, base.TestServerArgs{
+		// Note: We're only testing external-process mode because shared service
+		// mode tenants have all capabilities.
+		DefaultTestTenant: base.ExternalTestTenantAlwaysEnabled,
+	})
+	defer srv.Stopper().Stop(context.Background())
+
+	s := srv.ApplicationLayer()
+
+	var engineStats serverpb.EngineStatsResponse
+	// Using SucceedsSoon because we have seen in the wild that
+	// occasionally requests don't go through with error "transport:
+	// error while dialing: connection interrupted (did the remote node
+	// shut down or are there networking issues?)"
+	testutils.SucceedsSoon(t, func() error {
+		actualErr := srvtestutils.GetStatusJSONProto(s, "enginestats/local", &engineStats)
+		require.Error(t, actualErr)
+		if !strings.Contains(actualErr.Error(), "client tenant does not have capability to query cluster node metadata") {
+			return errors.Wrap(actualErr, "unexpected error message")
+		}
+		return nil
+	})
+
+}