Skip to content

Commit 3f285af

Browse files
craig[bot]stevendannakyle-a-wongshubhamdhamaDarrylWong
committed
121999: demo: remove system tenant DDL restrictions r=dt a=stevendanna This makes demo different from the current out of box experience where these settings. Epic: none Release note: None 153500: server: add txn bundle download; refactor handlers r=dhartunian a=kyle-a-wong Previously, handlers for statement diagnostic bundles were oddly placed in the gateway configuration, and their implementations were crowded into `admin.go`. This change moves the code into respective files containing status endpoint implementations (still in same package, unfortunately) in `statement_diagnostic_requests.go` and `transaction_diagnostic.requests.go`. Handlers are registered in `server_http.go` making them easier to find, and they use the go http mux to route instead of gRPC Gateway. We also implement a handler for download transaction diagnostic bundles. This handler constructs the internal statement bundles first, adding them to the transaction's zip file, and also includes the transaction's own bundle containing its full trace. The output of the new handler is a similar direct zip file download like the statement bundle. Epic: CRDB-53541 Resolves: CRDB-53547 Release note: None ---- Note for reviewers: Only review final commit, first 4 have separate PRs 154012: server,drpc: disable DRPC for REST-RPC bridge r=shubhamdhama a=shubhamdhama To prepare for the upcoming release, we are temporarily disabling DRPC for the REST-RPC bridge since some important work remains to be finished. - #152823 - #152821 - #152824 - #151398 This change includes disabling the new REST-RPC bridge. This means that grpc-gateway would serve requests even when DRPC is enabled. The other change is disabling DRPC RPC clients for statusServer and adminServer, since the work for forwarding authentication information is still in progress. Epic: none Release note: none Fixes: #154017 154194: roachtest: remove global-seed flag r=herkolategan a=DarrylWong This flag allows the user to seed the global rng instance for the entire test suite run. This flag is rarely (never) used and adds confusion for test writers as they are unsure if they should use the global seed or the test scoped COCKROACH_RANDOM_SEED. This changes the flag to be a ROACHTEST_GLOBAL_SEED env var which still allows it to be set, but unexposes it from being used directly in roachtests themselves. Fixes: none Epic: none Release note: none 154208: stmtdiagnostics: add 25.4 version gate to txn diagnostics polling r=kyle-a-wong a=kyle-a-wong Adds a version gate to the TxnRegistry.pollTxnRequests to return early if v25.4 is not active. This will ensure that the `system.transaction_diagnostics_requests` table exists before try to query it. Epic: None Release note: None 154209: rowexec: manually flush progress when backfiling vector indexes r=mw5h a=mw5h Previously, the vector index codepath in backfill took no special action to update progress tracking. Since progress tracking uses BulkAdder flushes to determine when to flush progress and vector indexes don't currently use BulkAdders during backfill, this meant that vector index build status never got updated. This patch adds a manual flush of status whenever an IndexBatch containing a vector index completes. Vector index inserts are pretty slow and IndexBatches are pretty large, so this is unlikely to lead to too many flushes. Fixes: #146691 Release note (bug fix): Index backfill will now properly track job progress in SHOW JOBS output. Co-authored-by: Steven Danna <[email protected]> Co-authored-by: Kyle Wong <[email protected]> Co-authored-by: Shubham Dhama <[email protected]> Co-authored-by: DarrylWong <[email protected]> Co-authored-by: Matt White <[email protected]>
7 parents f3c24cf + c19bd1f + c60ff89 + 7c1d849 + a189df7 + f4cf3d5 + a8d9df5 commit 3f285af

File tree

24 files changed

+499
-136
lines changed

24 files changed

+499
-136
lines changed

pkg/acceptance/localcluster/cluster.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -478,7 +478,7 @@ func (n *Node) StatusClient(ctx context.Context) serverpb.RPCStatusClient {
478478
return existingClient
479479
}
480480

481-
if !rpcbase.DRPCEnabled(ctx, n.rpcCtx.Settings) {
481+
if !rpcbase.TODODRPC || !rpcbase.DRPCEnabled(ctx, n.rpcCtx.Settings) {
482482
conn, err := n.rpcCtx.GRPCUnvalidatedDial(n.RPCAddr(), roachpb.Locality{}).Connect(ctx)
483483
if err != nil {
484484
log.Dev.Fatalf(context.Background(), "failed to initialize status client: %s", err)

pkg/cli/democluster/BUILD.bazel

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,6 @@ go_library(
3737
"//pkg/sql/catalog/catalogkeys",
3838
"//pkg/sql/isql",
3939
"//pkg/sql/sem/catconstants",
40-
"//pkg/sql/sqlclustersettings",
4140
"//pkg/storage/fs",
4241
"//pkg/testutils/serverutils",
4342
"//pkg/testutils/serverutils/regionlatency",
@@ -78,7 +77,6 @@ go_test(
7877
"//pkg/security/securityassets",
7978
"//pkg/security/securitytest",
8079
"//pkg/server",
81-
"//pkg/sql/sqlclustersettings",
8280
"//pkg/storage/fs",
8381
"//pkg/testutils",
8482
"//pkg/testutils/serverutils/regionlatency",

pkg/cli/democluster/demo_cluster.go

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,6 @@ import (
3939
"github.com/cockroachdb/cockroach/pkg/sql/catalog/catalogkeys"
4040
"github.com/cockroachdb/cockroach/pkg/sql/isql"
4141
"github.com/cockroachdb/cockroach/pkg/sql/sem/catconstants"
42-
"github.com/cockroachdb/cockroach/pkg/sql/sqlclustersettings"
4342
"github.com/cockroachdb/cockroach/pkg/storage/fs"
4443
"github.com/cockroachdb/cockroach/pkg/testutils/serverutils"
4544
"github.com/cockroachdb/cockroach/pkg/testutils/serverutils/regionlatency"
@@ -457,15 +456,6 @@ func (c *transientCluster) Start(ctx context.Context) (err error) {
457456
return err
458457
}
459458
}
460-
461-
for _, s := range []string{
462-
string(sqlclustersettings.RestrictAccessToSystemInterface.Name()),
463-
string(sql.TipUserAboutSystemInterface.Name()),
464-
} {
465-
if _, err := ie.Exec(ctx, "restrict-system-interface", nil, fmt.Sprintf(`SET CLUSTER SETTING %s = true`, s)); err != nil {
466-
return err
467-
}
468-
}
469459
}
470460

471461
// Prepare the URL for use by the SQL shell.

pkg/cli/democluster/demo_cluster_test.go

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@ import (
2222
"github.com/cockroachdb/cockroach/pkg/security/securityassets"
2323
"github.com/cockroachdb/cockroach/pkg/security/securitytest"
2424
"github.com/cockroachdb/cockroach/pkg/server"
25-
"github.com/cockroachdb/cockroach/pkg/sql/sqlclustersettings"
2625
"github.com/cockroachdb/cockroach/pkg/storage/fs"
2726
"github.com/cockroachdb/cockroach/pkg/testutils"
2827
"github.com/cockroachdb/cockroach/pkg/testutils/serverutils/regionlatency"
@@ -309,9 +308,6 @@ func TestTransientClusterMultitenant(t *testing.T) {
309308
ctx, cancel = c.stopper.WithCancelOnQuiesce(ctx)
310309
defer cancel()
311310

312-
// Ensure CREATE TABLE below works properly.
313-
sqlclustersettings.RestrictAccessToSystemInterface.Override(ctx, &c.firstServer.SystemLayer().ClusterSettings().SV, false)
314-
315311
testutils.RunTrueAndFalse(t, "forSecondaryTenant", func(t *testing.T, forSecondaryTenant bool) {
316312
url, err := c.getNetworkURLForServer(ctx, 0,
317313
true /* includeAppName */, serverSelection(forSecondaryTenant))

pkg/cmd/roachtest/roachtestflags/BUILD.bazel

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@ go_library(
1111
deps = [
1212
"//pkg/cmd/roachtest/spec",
1313
"//pkg/roachprod/vm",
14-
"//pkg/util/randutil",
1514
"@com_github_cockroachdb_errors//:errors",
1615
"@com_github_spf13_pflag//:pflag",
1716
],

pkg/cmd/roachtest/roachtestflags/flags.go

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@ import (
1212

1313
"github.com/cockroachdb/cockroach/pkg/cmd/roachtest/spec"
1414
"github.com/cockroachdb/cockroach/pkg/roachprod/vm"
15-
"github.com/cockroachdb/cockroach/pkg/util/randutil"
1615
"github.com/spf13/pflag"
1716
)
1817

@@ -476,12 +475,6 @@ var (
476475
Usage: `Percentage of failed tests before all remaining tests are automatically terminated.`,
477476
})
478477

479-
GlobalSeed int64 = randutil.NewPseudoSeed()
480-
_ = registerRunFlag(&GlobalSeed, FlagInfo{
481-
Name: "global-seed",
482-
Usage: `The global random seed used for all tests.`,
483-
})
484-
485478
ClearClusterCache bool = true
486479
_ = registerRunFlag(&ClearClusterCache, FlagInfo{
487480
Name: "clear-cluster-cache",

pkg/cmd/roachtest/run.go

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ import (
1515
"os/user"
1616
"path/filepath"
1717
"runtime"
18+
"strconv"
1819
"strings"
1920
"time"
2021

@@ -30,6 +31,7 @@ import (
3031
"github.com/cockroachdb/cockroach/pkg/roachprod/logger"
3132
"github.com/cockroachdb/cockroach/pkg/util/allstacks"
3233
"github.com/cockroachdb/cockroach/pkg/util/leaktest"
34+
"github.com/cockroachdb/cockroach/pkg/util/randutil"
3335
"github.com/cockroachdb/cockroach/pkg/util/stop"
3436
"github.com/cockroachdb/cockroach/pkg/util/timeutil"
3537
"github.com/cockroachdb/errors"
@@ -50,8 +52,16 @@ const (
5052
// runTests is the main function for the run and bench commands.
5153
// Assumes initRunFlagsBinariesAndLibraries was called.
5254
func runTests(register func(registry.Registry), filter *registry.TestFilter) error {
55+
globalSeed := randutil.NewPseudoSeed()
56+
if globalSeedEnv := os.Getenv("ROACHTEST_GLOBAL_SEED"); globalSeedEnv != "" {
57+
if parsed, err := strconv.ParseInt(globalSeedEnv, 0, 64); err == nil {
58+
globalSeed = parsed
59+
} else {
60+
return errors.Wrapf(err, "could not parse ROACHTEST_GLOBAL_SEED=%q", globalSeedEnv)
61+
}
62+
}
5363
//lint:ignore SA1019 deprecated
54-
rand.Seed(roachtestflags.GlobalSeed)
64+
rand.Seed(globalSeed)
5565
r := makeTestRegistry()
5666

5767
// actual registering of tests
@@ -147,7 +157,7 @@ func runTests(register func(registry.Registry), filter *registry.TestFilter) err
147157
teamLoader: team.DefaultLoadTeams,
148158
}
149159

150-
l.Printf("global random seed: %d", roachtestflags.GlobalSeed)
160+
l.Printf("global random seed: %d", globalSeed)
151161
go func() {
152162
if err := http.ListenAndServe(
153163
fmt.Sprintf(":%d", roachtestflags.PromPort),

pkg/rpc/rpcbase/nodedialer.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,10 @@ var ExperimentalDRPCEnabled = settings.RegisterBoolSetting(
2727
"if true, use drpc to execute Batch RPCs (instead of gRPC)",
2828
envExperimentalDRPCEnabled)
2929

30+
// TODODRPC is a marker to identify sites that needs to be updated to support
31+
// DRPC.
32+
const TODODRPC = false
33+
3034
// NodeDialer interface defines methods for dialing peer nodes using their
3135
// node IDs.
3236
type NodeDialer interface {

pkg/server/BUILD.bazel

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -330,6 +330,7 @@ go_library(
330330
"//pkg/util/log/logmetrics",
331331
"//pkg/util/log/logpb",
332332
"//pkg/util/log/severity",
333+
"//pkg/util/memzipper",
333334
"//pkg/util/metric",
334335
"//pkg/util/mon",
335336
"//pkg/util/netutil",
@@ -377,7 +378,6 @@ go_library(
377378
"@com_github_google_pprof//profile",
378379
"@com_github_gorilla_mux//:mux",
379380
"@com_github_grpc_ecosystem_grpc_gateway//runtime:go_default_library",
380-
"@com_github_grpc_ecosystem_grpc_gateway//utilities:go_default_library",
381381
"@com_github_lib_pq//:pq",
382382
"@com_github_marusama_semaphore//:semaphore",
383383
"@com_github_nightlyone_lockfile//:lockfile",

pkg/server/admin.go

Lines changed: 0 additions & 92 deletions
Original file line numberDiff line numberDiff line change
@@ -6,12 +6,9 @@
66
package server
77

88
import (
9-
"bytes"
109
"context"
1110
"encoding/json"
1211
"fmt"
13-
"io"
14-
"net/http"
1512
"sort"
1613
"strconv"
1714
"strings"
@@ -77,10 +74,8 @@ import (
7774
"github.com/cockroachdb/errors"
7875
"github.com/cockroachdb/redact"
7976
gwruntime "github.com/grpc-ecosystem/grpc-gateway/runtime"
80-
gwutil "github.com/grpc-ecosystem/grpc-gateway/utilities"
8177
"google.golang.org/grpc"
8278
"google.golang.org/grpc/codes"
83-
"google.golang.org/grpc/metadata"
8479
grpcstatus "google.golang.org/grpc/status"
8580
"storj.io/drpc"
8681
"storj.io/drpc/drpcerr"
@@ -280,44 +275,6 @@ func (s *adminServer) RegisterDRPCService(d drpc.Mux) error {
280275
func (s *adminServer) RegisterGateway(
281276
ctx context.Context, mux *gwruntime.ServeMux, conn *grpc.ClientConn,
282277
) error {
283-
// Register the /_admin/v1/stmtbundle endpoint, which serves statement support
284-
// bundles as files.
285-
stmtBundlePattern := gwruntime.MustPattern(gwruntime.NewPattern(
286-
1, /* version */
287-
[]int{
288-
int(gwutil.OpLitPush), 0, int(gwutil.OpLitPush), 1, int(gwutil.OpLitPush), 2,
289-
int(gwutil.OpPush), 0, int(gwutil.OpConcatN), 1, int(gwutil.OpCapture), 3},
290-
[]string{"_admin", "v1", "stmtbundle", "id"},
291-
"", /* verb */
292-
))
293-
294-
mux.Handle("GET", stmtBundlePattern, func(
295-
w http.ResponseWriter, req *http.Request, pathParams map[string]string,
296-
) {
297-
idStr, ok := pathParams["id"]
298-
if !ok {
299-
http.Error(w, "missing id", http.StatusBadRequest)
300-
return
301-
}
302-
id, err := strconv.ParseInt(idStr, 10, 64)
303-
if err != nil {
304-
http.Error(w, "invalid id", http.StatusBadRequest)
305-
return
306-
}
307-
308-
// The privilege checks in the privilege checker below checks the user in the incoming
309-
// gRPC metadata.
310-
md := authserver.TranslateHTTPAuthInfoToGRPCMetadata(req.Context(), req)
311-
authCtx := metadata.NewIncomingContext(req.Context(), md)
312-
authCtx = s.AnnotateCtx(authCtx)
313-
if err := s.privilegeChecker.RequireViewActivityAndNoViewActivityRedactedPermission(authCtx); err != nil {
314-
http.Error(w, err.Error(), http.StatusForbidden)
315-
return
316-
}
317-
s.getStatementBundle(req.Context(), id, w)
318-
})
319-
320-
// Register the endpoints defined in the proto.
321278
return serverpb.RegisterAdminHandler(ctx, mux, conn)
322279
}
323280

@@ -2646,55 +2603,6 @@ func (s *adminServer) QueryPlan(
26462603
}, nil
26472604
}
26482605

2649-
// getStatementBundle retrieves the statement bundle with the given id and
2650-
// writes it out as an attachment. Note this function assumes the user has
2651-
// permission to access the statement bundle.
2652-
func (s *adminServer) getStatementBundle(ctx context.Context, id int64, w http.ResponseWriter) {
2653-
row, err := s.internalExecutor.QueryRowEx(
2654-
ctx, "admin-stmt-bundle", nil, /* txn */
2655-
sessiondata.NodeUserSessionDataOverride,
2656-
"SELECT bundle_chunks FROM system.statement_diagnostics WHERE id=$1 AND bundle_chunks IS NOT NULL",
2657-
id,
2658-
)
2659-
if err != nil {
2660-
http.Error(w, err.Error(), http.StatusInternalServerError)
2661-
return
2662-
}
2663-
if row == nil {
2664-
http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
2665-
return
2666-
}
2667-
// Put together the entire bundle. Ideally we would stream it in chunks,
2668-
// but it's hard to return errors once we start.
2669-
var bundle bytes.Buffer
2670-
chunkIDs := row[0].(*tree.DArray).Array
2671-
for _, chunkID := range chunkIDs {
2672-
chunkRow, err := s.internalExecutor.QueryRowEx(
2673-
ctx, "admin-stmt-bundle", nil, /* txn */
2674-
sessiondata.NodeUserSessionDataOverride,
2675-
"SELECT data FROM system.statement_bundle_chunks WHERE id=$1",
2676-
chunkID,
2677-
)
2678-
if err != nil {
2679-
http.Error(w, err.Error(), http.StatusInternalServerError)
2680-
return
2681-
}
2682-
if chunkRow == nil {
2683-
http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
2684-
return
2685-
}
2686-
data := chunkRow[0].(*tree.DBytes)
2687-
bundle.WriteString(string(*data))
2688-
}
2689-
2690-
w.Header().Set(
2691-
"Content-Disposition",
2692-
fmt.Sprintf("attachment; filename=stmt-bundle-%d.zip", id),
2693-
)
2694-
2695-
_, _ = io.Copy(w, &bundle)
2696-
}
2697-
26982606
// DecommissionPreCheck runs checks and returns the DecommissionPreCheckResponse
26992607
// for the given nodes.
27002608
func (s *systemAdminServer) DecommissionPreCheck(

0 commit comments

Comments
 (0)