sql: use a random minute for the sql-stats-compaction job default recurrence

Now, the sql-stats-compaction job that is created during cluster
initialization will be scheduled on a random minute in the hour, rather
than at the top of the hour. This will only affect clusters that are
initialized after this change is released. Any existing clusters will
continue to keep whatever recurrence they had before, which defaulted to
@hourly.

This change was made because we have observed that this job can cause
CPU spikes on the serverless host clusters, since different tenants all
had this job scheduled for the same time.

Release note: None

Author: rafiss

pkg/sql/conn_executor.go

@@ -471,6 +471,7 @@ func NewServer(cfg *ExecutorConfig, pool *mon.BytesMonitor) *Server {
 		DB: NewInternalDB(
 			s, MemoryMetrics{}, sqlStatsInternalExecutorMonitor,
 		),
+		ClusterID:      s.cfg.NodeInfo.LogicalClusterID,
 		SQLIDContainer: cfg.NodeInfo.NodeID,
 		JobRegistry:    s.cfg.JobRegistry,
 		Knobs:          cfg.SQLStatsTestingKnobs,

pkg/sql/sqlstats/persistedsqlstats/BUILD.bazel

@@ -49,6 +49,7 @@ go_library(
         "//pkg/util/stop",
         "//pkg/util/syncutil",
         "//pkg/util/timeutil",
+        "//pkg/util/uuid",
         "@com_github_cockroachdb_errors//:errors",
         "@com_github_gogo_protobuf//types",
         "@com_github_robfig_cron_v3//:cron",

pkg/sql/sqlstats/persistedsqlstats/compaction_scheduling.go

@@ -21,6 +21,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/sql/isql"
 	"github.com/cockroachdb/cockroach/pkg/sql/sem/tree"
 	"github.com/cockroachdb/cockroach/pkg/sql/sessiondata"
+	"github.com/cockroachdb/cockroach/pkg/util/uuid"
 	"github.com/cockroachdb/errors"
 	pbtypes "github.com/gogo/protobuf/types"
 )
@@ -35,7 +36,7 @@ var ErrDuplicatedSchedules = errors.New("creating multiple sql stats compaction
 // scheduled job subsystem so the compaction job can be run periodically. This
 // is done during the cluster startup upgrade.
 func CreateSQLStatsCompactionScheduleIfNotYetExist(
-	ctx context.Context, txn isql.Txn, st *cluster.Settings,
+	ctx context.Context, txn isql.Txn, st *cluster.Settings, clusterID uuid.UUID,
 ) (*jobs.ScheduledJob, error) {
 	scheduleExists, err := checkExistingCompactionSchedule(ctx, txn)
 	if err != nil {
@@ -48,7 +49,9 @@ func CreateSQLStatsCompactionScheduleIfNotYetExist(
 
 	compactionSchedule := jobs.NewScheduledJob(scheduledjobs.ProdJobSchedulerEnv)
 
-	schedule := SQLStatsCleanupRecurrence.Get(&st.SV)
+	schedule := scheduledjobs.MaybeRewriteCronExpr(
+		clusterID, SQLStatsCleanupRecurrence.Get(&st.SV),
+	)
 	if err := compactionSchedule.SetSchedule(schedule); err != nil {
 		return nil, err
 	}

pkg/sql/sqlstats/persistedsqlstats/controller.go

@@ -19,6 +19,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/sql/isql"
 	"github.com/cockroachdb/cockroach/pkg/sql/sessiondata"
 	"github.com/cockroachdb/cockroach/pkg/sql/sqlstats/sslocal"
+	"github.com/cockroachdb/cockroach/pkg/util/uuid"
 )
 
 // Controller implements the SQL Stats subsystem control plane. This exposes
@@ -27,8 +28,9 @@ import (
 // subsystem.
 type Controller struct {
 	*sslocal.Controller
-	db isql.DB
-	st *cluster.Settings
+	db        isql.DB
+	st        *cluster.Settings
+	clusterID func() uuid.UUID
 }
 
 // NewController returns a new instance of sqlstats.Controller.
@@ -39,14 +41,15 @@ func NewController(
 		Controller: sslocal.NewController(sqlStats.SQLStats, status),
 		db:         db,
 		st:         sqlStats.cfg.Settings,
+		clusterID:  sqlStats.cfg.ClusterID,
 	}
 }
 
 // CreateSQLStatsCompactionSchedule implements the tree.SQLStatsController
 // interface.
 func (s *Controller) CreateSQLStatsCompactionSchedule(ctx context.Context) error {
 	return s.db.Txn(ctx, func(ctx context.Context, txn isql.Txn) error {
-		_, err := CreateSQLStatsCompactionScheduleIfNotYetExist(ctx, txn, s.st)
+		_, err := CreateSQLStatsCompactionScheduleIfNotYetExist(ctx, txn, s.st, s.clusterID())
 		return err
 	})
 }

pkg/sql/sqlstats/persistedsqlstats/provider.go

@@ -33,13 +33,15 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/util/stop"
 	"github.com/cockroachdb/cockroach/pkg/util/syncutil"
 	"github.com/cockroachdb/cockroach/pkg/util/timeutil"
+	"github.com/cockroachdb/cockroach/pkg/util/uuid"
 )
 
 // Config is a configuration struct for the persisted SQL stats subsystem.
 type Config struct {
 	Settings                *cluster.Settings
 	InternalExecutorMonitor *mon.BytesMonitor
 	DB                      isql.DB
+	ClusterID               func() uuid.UUID
 	SQLIDContainer          *base.SQLIDContainer
 	JobRegistry             *jobs.Registry
 
@@ -100,6 +102,7 @@ func New(cfg *Config, memSQLStats *sslocal.SQLStats) *PersistedSQLStats {
 
 	p.jobMonitor = jobMonitor{
 		st:           cfg.Settings,
+		clusterID:    cfg.ClusterID,
 		db:           cfg.DB,
 		scanInterval: defaultScanInterval,
 		jitterFn:     p.jitterInterval,

pkg/sql/sqlstats/persistedsqlstats/scheduled_job_monitor.go

@@ -25,6 +25,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/util/retry"
 	"github.com/cockroachdb/cockroach/pkg/util/stop"
 	"github.com/cockroachdb/cockroach/pkg/util/timeutil"
+	"github.com/cockroachdb/cockroach/pkg/util/uuid"
 	"github.com/cockroachdb/errors"
 )
 
@@ -60,6 +61,7 @@ var longIntervalWarningThreshold = time.Hour * 24
 // periodically every scanInterval (subject to jittering).
 type jobMonitor struct {
 	st           *cluster.Settings
+	clusterID    func() uuid.UUID
 	db           isql.DB
 	scanInterval time.Duration
 	jitterFn     func(time.Duration) time.Duration
@@ -171,7 +173,7 @@ func (j *jobMonitor) updateSchedule(ctx context.Context, cronExpr string) {
 				if !jobs.HasScheduledJobNotFoundError(err) && !errors.Is(err, errScheduleNotFound) {
 					return err
 				}
-				sj, err = CreateSQLStatsCompactionScheduleIfNotYetExist(ctx, txn, j.st)
+				sj, err = CreateSQLStatsCompactionScheduleIfNotYetExist(ctx, txn, j.st, j.clusterID())
 				if err != nil {
 					return err
 				}