vecindex: fix determinism violations

Fix a couple cases where the vector index is not behaving
deterministically even when configured to do so. In one case, partition
keys were not generated deterministically. In the other case, the seed
used in crdb_test builds was different than the seed used otherwise.

Epic: CRDB-42943
Release note: None

Author: andy-kimball

pkg/sql/vecindex/manager.go

@@ -19,7 +19,6 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/sql/vecindex/cspann/quantize"
 	"github.com/cockroachdb/cockroach/pkg/sql/vecindex/vecpb"
 	"github.com/cockroachdb/cockroach/pkg/sql/vecindex/vecstore"
-	"github.com/cockroachdb/cockroach/pkg/util/buildutil"
 	"github.com/cockroachdb/cockroach/pkg/util/log"
 	"github.com/cockroachdb/cockroach/pkg/util/metric"
 	"github.com/cockroachdb/cockroach/pkg/util/stop"
@@ -136,7 +135,8 @@ func (m *Manager) Get(
 		}
 		// TODO(drewk): use the config to populate the index options as well.
 		quantizer := quantize.NewRaBitQuantizer(int(config.Dims), config.Seed, config.DistanceMetric)
-		store, err := vecstore.New(ctx, m.db, quantizer, m.codec, tableID, indexID)
+		store, err := vecstore.New(
+			ctx, m.db, quantizer, m.codec, tableID, indexID, config.IsDeterministic)
 		if err != nil {
 			return nil, err
 		}
@@ -220,13 +220,6 @@ func (m *Manager) getVecConfig(
 	if config.Dims <= 0 {
 		return vecpb.Config{}, errInvalidVecConfig
 	}
-	// TODO(mw5h, drewk): this should be a session setting in create index rather
-	// than an override like this.
-	if buildutil.CrdbTestBuild {
-		// This is a test build, so let's use a fixed seed for the random projection to
-		// avoid test flakes.
-		config.Seed = 0xdeadcafe
-	}
 	return config, nil
 }
 

pkg/sql/vecindex/vecstore/store.go

@@ -8,6 +8,7 @@ package vecstore
 import (
 	"context"
 	"slices"
+	"sync/atomic"
 
 	"github.com/cockroachdb/cockroach/pkg/keys"
 	"github.com/cockroachdb/cockroach/pkg/kv"
@@ -38,6 +39,12 @@ type Store struct {
 	// readOnly is true if the store does not accept writes.
 	readOnly bool
 
+	// isDeterministic is true if the store should generate new partition keys
+	// using a deterministic algorithm.
+	isDeterministic bool
+	// keyGen generates new partition keys deterministically.
+	keyGen atomic.Uint64
+
 	codec   keys.SQLCodec
 	tableID catid.DescID
 	indexID catid.IndexID
@@ -98,6 +105,7 @@ func New(
 	defaultCodec keys.SQLCodec,
 	tableID catid.DescID,
 	indexID catid.IndexID,
+	isDeterministic bool,
 ) (ps *Store, err error) {
 	ps = &Store{
 		db:             db,
@@ -110,6 +118,10 @@ func New(
 		minConsistency: kvpb.INCONSISTENT,
 		emptyVec:       make(vector.T, quantizer.GetDims()),
 	}
+	if isDeterministic {
+		ps.isDeterministic = true
+		ps.keyGen.Store(1)
+	}
 
 	err = db.DescsTxn(ctx, func(ctx context.Context, txn descs.Txn) error {
 		tableDesc, err := txn.Descriptors().ByIDWithLeased(txn.KV()).Get().Table(ctx, tableID)
@@ -196,6 +208,11 @@ func (s *Store) RunTransaction(ctx context.Context, fn func(txn cspann.Txn) erro
 // MakePartitionKey is part of the cspann.Store interface. It allocates a new
 // unique partition key.
 func (s *Store) MakePartitionKey() cspann.PartitionKey {
+	if s.isDeterministic {
+		// Generate new partition keys by incrementing an atomic.
+		return cspann.PartitionKey(s.keyGen.Add(1))
+	}
+
 	instanceID := s.kv.Context().NodeID.SQLInstanceID()
 	return cspann.PartitionKey(unique.GenerateUniqueUnorderedID(unique.ProcessUniqueID(instanceID)))
 }