sql: estimate table statistic staleness in stats.Refresher

This commit adds an `EstimateStaleness()` method to the table statistic
`Refresher`, which estimates the current fraction of stale rows in a
given table with the formula:

`cur_fraction_stale = (time_since_last_refresh /
avg_time_between_refreshes) * target_fraction_stale_rows`

Although this isn’t used anywhere yet, it will be useful for logging
when scans are misestimated (see #153748).

Part of: #153748, #153873

Release note: None

Author: Uzair5162

pkg/sql/stats/automatic_stats.go

@@ -340,6 +340,9 @@ type TableStatsTestingKnobs struct {
 	// perform full statistics refreshes. Useful for testing the partial stats
 	// refresh logic.
 	DisableFullStatsRefresh bool
+	// StubTimeNow allows tests to override the current time, used by
+	// EstimateStaleness to get the latest stats' age.
+	StubTimeNow func() time.Time
 }
 
 var _ base.ModuleTestingKnobs = &TableStatsTestingKnobs{}
@@ -893,6 +896,56 @@ func (r *Refresher) NotifyMutation(table catalog.TableDescriptor, rowsAffected i
 	}
 }
 
+// EstimateStaleness returns an estimate fraction of stale rows in the given
+// table based on how long it has been since the last full statistics refresh,
+// and the average time between refreshes.
+func (r *Refresher) EstimateStaleness(ctx context.Context, tableID descpb.ID) (float64, error) {
+	desc := r.getTableDescriptor(ctx, tableID)
+	if desc == nil {
+		return 0, errors.New("could not access the table descriptor")
+	}
+	if !autostatsCollectionAllowed(desc, r.st) {
+		return 0, errors.New("automatic stats collection is not allowed for this table")
+	}
+
+	var forecast *bool
+	// NB: we pass nil boolean as 'forecast' argument in order to not invalidate
+	// the stats cache entry since we don't care whether there is a forecast or
+	// not in the stats.
+	tableStats, err := r.cache.getTableStatsFromCache(ctx, tableID, forecast, nil /* udtCols */, nil /* typeResolver */)
+	if err != nil {
+		return 0, err
+	}
+
+	// Find the most recent full statistic
+	var stat *TableStatistic
+	for _, s := range tableStats {
+		if !s.IsPartial() && !s.IsForecast() && !s.IsMerged() {
+			stat = s
+			break
+		}
+	}
+	if stat == nil {
+		return 0, errors.New("no full statistics available")
+	}
+
+	var explicitSettings *catpb.AutoStatsSettings
+	if s, ok := r.settingOverrides[tableID]; ok {
+		explicitSettings = &s
+	}
+	staleTargetFraction := r.autoStatsFractionStaleRows(explicitSettings)
+
+	avgRefreshTime := avgFullRefreshTime(tableStats)
+	statsAge := timeutil.Since(stat.CreatedAt)
+	if r.knobs != nil && r.knobs.StubTimeNow != nil {
+		statsAge = r.knobs.StubTimeNow().Sub(stat.CreatedAt)
+	}
+	staleFraction :=
+		float64(statsAge) / float64(avgRefreshTime) * staleTargetFraction
+
+	return staleFraction, nil
+}
+
 // maybeRefreshStats implements the core logic described in the comment for
 // Refresher. It is called by the background Refresher thread.
 // explicitSettings, if non-nil, holds any autostats cluster setting overrides

pkg/sql/stats/automatic_stats_test.go

@@ -1063,3 +1063,190 @@ func TestRefresherReadOnlyShutdown(t *testing.T) {
 	// Wait for shutdown - this should complete without hanging.
 	readOnlyRefresher.WaitForAutoStatsShutdown(ctx)
 }
+
+func TestEstimateStaleness(t *testing.T) {
+	defer leaktest.AfterTest(t)()
+	defer log.Scope(t).Close(t)
+	ctx := context.Background()
+
+	srv, sqlDB, _ := serverutils.StartServer(t, base.TestServerArgs{})
+	defer srv.Stopper().Stop(ctx)
+	s := srv.ApplicationLayer()
+	codec, st := s.Codec(), s.ClusterSettings()
+
+	evalCtx := eval.NewTestingEvalContext(st)
+	defer evalCtx.Stop(ctx)
+
+	AutomaticStatisticsClusterMode.Override(ctx, &st.SV, false)
+
+	sqlRun := sqlutils.MakeSQLRunner(sqlDB)
+	sqlRun.Exec(t,
+		`CREATE DATABASE t;
+		CREATE TABLE t.a (k INT PRIMARY KEY);
+		INSERT INTO t.a VALUES (1);`)
+
+	internalDB := s.InternalDB().(descs.DB)
+	table := desctestutils.TestingGetPublicTableDescriptor(s.DB(), codec, "t", "a")
+	cache := NewTableStatisticsCache(
+		10, /* cacheSize */
+		s.ClusterSettings(),
+		s.InternalDB().(descs.DB),
+		s.AppStopper(),
+	)
+	require.NoError(t, cache.Start(ctx, codec, s.RangeFeedFactory().(*rangefeed.Factory)))
+
+	// curTime is used as the current time throughout the test to ensure that the
+	// calculated staleness is consistent even if there are delays due to
+	// running the test under race.
+	curTime := timeutil.Now().Round(time.Hour)
+	knobs := &TableStatsTestingKnobs{
+		StubTimeNow: func() time.Time { return curTime },
+	}
+	refresher := MakeRefresher(s.AmbientCtx(), st, internalDB, cache, time.Microsecond /* asOfTime */, knobs, false /* readOnlyTenant */)
+
+	checkEstimatedStaleness := func(expected float64) error {
+		return testutils.SucceedsSoonError(func() error {
+			actual, err := refresher.EstimateStaleness(ctx,
+				table.GetID())
+			if err != nil {
+				return err
+			}
+			if actual != expected {
+				return fmt.Errorf("expected EstimateStaleness %f but found %f",
+					expected, actual)
+			}
+			return nil
+		})
+	}
+
+	insertStat := func(
+		txn *kv.Txn, name string, columnIDs *tree.DArray, createdAt *tree.DTimestamp,
+	) error {
+		_, err := internalDB.Executor().Exec(
+			ctx, "insert-statistic", txn,
+			`INSERT INTO system.table_statistics (
+					  "tableID",
+					  "name",
+					  "columnIDs",
+					  "createdAt",
+					  "rowCount",
+					  "distinctCount",
+					  "nullCount",
+					  "avgSize"
+				  ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`,
+			table.GetID(),
+			name,
+			columnIDs,
+			createdAt,
+			100000, /* rowCount */
+			1,      /* distinctCount */
+			0,      /* nullCount */
+			4,      /* avgSize */
+		)
+		return err
+	}
+
+	overwriteFullStats := func(startOffsetHours, intervalHours int) error {
+		return s.DB().Txn(ctx, func(ctx context.Context, txn *kv.Txn) error {
+			_, err := internalDB.Executor().Exec(
+				ctx, "delete-stats", txn,
+				`DELETE FROM system.table_statistics WHERE "tableID" = $1`,
+				table.GetID(),
+			)
+			if err != nil {
+				return err
+			}
+
+			for i := 0; i < 5; i++ {
+				columnIDsVal := tree.NewDArray(types.Int)
+				if err := columnIDsVal.Append(tree.NewDInt(tree.DInt(1))); err != nil {
+					return err
+				}
+				offset := startOffsetHours + i*intervalHours
+				createdAt, err := tree.MakeDTimestamp(
+					curTime.Add(time.Duration(-offset)*time.Hour), time.Hour,
+				)
+				if err != nil {
+					return err
+				}
+				if err := insertStat(txn, jobspb.AutoStatsName, columnIDsVal, createdAt); err != nil {
+					return err
+				}
+			}
+			return nil
+		})
+	}
+
+	// Ensure that we return an error if estimating staleness without any stats.
+	_, err := refresher.EstimateStaleness(ctx, table.GetID())
+	require.Error(t, err)
+	require.Contains(t, err.Error(), "no full statistics available")
+
+	// Ensure that we return an error if estimating staleness on a table that
+	// doesn't allow auto stats.
+	descTableStats := desctestutils.TestingGetPublicTableDescriptor(s.DB(),
+		codec, "system", "table_statistics")
+	_, err = refresher.EstimateStaleness(ctx, descTableStats.GetID())
+	require.Error(t, err)
+	require.Contains(t, err.Error(), "automatic stats collection is not allowed for this table")
+
+	// Create stats with 10-hour intervals, the most recent being 5 hours old.
+	if err = overwriteFullStats(
+		5,  /* startOffsetHours */
+		10, /* intervalHours */
+	); err != nil {
+		t.Fatal(err)
+	}
+
+	// With default settings (fraction_stale_rows = 0.2) and the latest full stat
+	// being 5 hours old (half of avgRefreshTime of 10 hours), we expect 10%
+	// staleness.
+	if err = checkEstimatedStaleness(0.1); err != nil {
+		t.Fatal(err)
+	}
+
+	fractionStaleRows := 0.4
+	explicitSettings := catpb.AutoStatsSettings{FractionStaleRows: &fractionStaleRows}
+	refresher.settingOverrides[table.GetID()] = explicitSettings
+
+	// With fraction_stale_rows = 0.4 and the latest full stat being 5 hours old
+	// (half of avgRefreshTime of 10 hours), we expect 20% staleness.
+	if err = checkEstimatedStaleness(0.2); err != nil {
+		t.Fatal(err)
+	}
+
+	// Reset fraction_stale_rows to default (0.2)
+	delete(refresher.settingOverrides, table.GetID())
+
+	// Delete old stats and create stats with 3-hour intervals, the most recent
+	// being 15 hours old.
+	if err = overwriteFullStats(
+		15, /* startOffsetHours */
+		3,  /* intervalHours */
+	); err != nil {
+		t.Fatal(err)
+	}
+
+	// With default settings (fraction_stale_rows = 0.2) and the latest full stat
+	// being 15 hours old (5 times the avgRefreshTime of 3 hours), we expect 100%
+	// staleness.
+	if err = checkEstimatedStaleness(1.0); err != nil {
+		t.Fatal(err)
+	}
+
+	// Delete old stats and create stats with 2-hour intervals, the most recent
+	// being 15 hours old.
+	if err = overwriteFullStats(
+		15, /* startOffsetHours */
+		2,  /* intervalHours */
+	); err != nil {
+		t.Fatal(err)
+	}
+
+	// With default settings (fraction_stale_rows = 0.2) and the latest full stat
+	// being 15 hours old (7.5 times the avgRefreshTime of 2 hours), we expect
+	// 150% staleness.
+	if err = checkEstimatedStaleness(1.5); err != nil {
+		t.Fatal(err)
+	}
+}