Merge #143250

143250: vecindex: add Try methods to Store r=drewkimball a=andy-kimball

Add a new set of Try methods to the Store method that run outside the scope of a transaction. Each Try method checks one or two preconditions, and will only perform its operation if the preconditions pass. These new methods will be used to do background fixup processing without the use of transactions, in order to reduce contention between foreground and background work.

In order to handle multiple racing workers, this commit introduces a new state machine that tracks the progress of split and merge fixups. The current state of a partition is stored in its metadata. Each Try method takes "expected" metadata that represents a worker's current understanding of a partition's state. If the actual metadata does not match the expected metadata, then another worker must have updated it, and so the Try method fails with a ConditionFailedError. The calling worker can reset its understanding of that partition's state, and continue processing the fixup.

Epic: CRDB-42943

Release note: None

Co-authored-by: Andrew Kimball <[email protected]>

Author: craig[bot]

pkg/cmd/vecbench/mem_provider.go

@@ -233,13 +233,14 @@ func (m *MemProvider) ensureIndex(ctx context.Context) error {
 	if m.index != nil {
 		return nil
 	}
+
+	quantizer := quantize.NewRaBitQuantizer(m.dims, seed)
 	if m.store == nil {
 		// Construct empty store if one doesn't yet exist.
-		m.store = memstore.New(m.dims, seed)
+		m.store = memstore.New(quantizer, seed)
 	}
 
 	var err error
-	quantizer := quantize.NewRaBitQuantizer(m.dims, seed)
 	m.index, err = cspann.NewIndex(ctx, m.store, quantizer, seed, &m.options, m.stopper)
 	return err
 }

pkg/sql/vecindex/cspann/BUILD.bazel

@@ -19,9 +19,11 @@ go_library(
         "kmeans.go",
         "pacer.go",
         "partition.go",
+        "partition_metadata.go",
         "search_set.go",
         "split_data.go",
         "store.go",
+        "store_errors.go",
     ],
     embed = [":cspann_go_proto"],
     importpath = "github.com/cockroachdb/cockroach/pkg/sql/vecindex/cspann",
@@ -52,6 +54,7 @@ go_test(
         "index_test.go",
         "kmeans_test.go",
         "pacer_test.go",
+        "partition_metadata_test.go",
         "partition_test.go",
         "search_set_test.go",
     ],

pkg/sql/vecindex/cspann/commontest/BUILD.bazel

@@ -14,6 +14,7 @@ go_library(
         "//pkg/sql/vecindex/cspann/testutils",
         "//pkg/sql/vecindex/cspann/workspace",
         "//pkg/util/num32",
+        "//pkg/util/timeutil",
         "//pkg/util/vector",
         "@com_github_cockroachdb_errors//:errors",
         "@com_github_stretchr_testify//require",