Merge #145018 #145538 #145901

145018: vecann: add new approximate nearest neighbor vector workload r=drewkimball a=andy-kimball

vecann is a new workload that tests vector indexes - both inserting vectors into them and searching them for approximate nearest neighbors. We can use this for load and scale testing.

The scenario is very simple - insert vectors from one of the datasets we've stored in a GCP cloud bucket and search over them. Similar to KV, the percentage of searches can be specified. The schema is also simple - a single table containing group id and an embedded vector. The group column is the prefix column for the vector index, so that a separate K-means tree is created for each unique group of vectors. This simulates a common case where companies want to insert and search embedding vectors for each of their customers in isolation.

Epic: CRDB-42943
Release note: None

145538: storepool: prevent holding multiple mutexes in UpdateLocalStoresAfterLeaseTransfer r=iskettaneh a=iskettaneh

This commit mitigates a deadlock that could potentially happen if two calls to UpdateLocalStoresAfterLeaseTransfer happen simultaneously in different order -> Leading to a deadlock.

Fixes: #145531

Release note: None

145901: workflow: auto-merge-backports should skip blathers PRs r=celiala,lunevalex a=rail

Previously, the search included PRs that are created by blathers. These PRs cannot be approved by blathers, so they should be skipped.

Release note: none
Epic: none

Co-authored-by: Andrew Kimball <[email protected]>
Co-authored-by: Ibrahim Kettaneh <[email protected]>
Co-authored-by: Rail Aliiev <[email protected]>

Author: 4 people

.github/workflows/auto-merge-backports.yml

@@ -24,7 +24,7 @@ jobs:
 
             const query = `
               query SearchPRs() {
-                search(query: "repo:${context.repo.owner}/${context.repo.repo} is:pr is:open label:backport label:blathers-backport label:backport-test-only", type: ISSUE, first: 100) {
+                search(query: "repo:${context.repo.owner}/${context.repo.repo} is:pr is:open label:backport label:blathers-backport label:backport-test-only -author:blathers-crl[bot]", type: ISSUE, first: 100) {
                   nodes {
                     ... on PullRequest {
                       number

pkg/BUILD.bazel

@@ -828,6 +828,7 @@ ALL_TESTS = [
     "//pkg/workload/sqlstats:sqlstats_test",
     "//pkg/workload/tpcc:tpcc_test",
     "//pkg/workload/tpch:tpch_test",
+    "//pkg/workload/vecann:vecann_test",
     "//pkg/workload/workloadimpl:workloadimpl_test",
     "//pkg/workload/workloadsql:workloadsql_test",
     "//pkg/workload/ycsb:ycsb_test",
@@ -2816,6 +2817,8 @@ GO_TARGETS = [
     "//pkg/workload/tpch:tpch_test",
     "//pkg/workload/ttlbench:ttlbench",
     "//pkg/workload/ttllogger:ttllogger",
+    "//pkg/workload/vecann:vecann",
+    "//pkg/workload/vecann:vecann_test",
     "//pkg/workload/workloadimpl:workloadimpl",
     "//pkg/workload/workloadimpl:workloadimpl_test",
     "//pkg/workload/workloadsql:workloadsql",

pkg/ccl/workloadccl/allccl/BUILD.bazel

@@ -31,6 +31,7 @@ go_library(
         "//pkg/workload/tpch",
         "//pkg/workload/ttlbench",
         "//pkg/workload/ttllogger",
+        "//pkg/workload/vecann",
         "//pkg/workload/ycsb",
     ],
 )

pkg/ccl/workloadccl/allccl/all.go

@@ -35,5 +35,6 @@ import (
 	_ "github.com/cockroachdb/cockroach/pkg/workload/tpch"
 	_ "github.com/cockroachdb/cockroach/pkg/workload/ttlbench"
 	_ "github.com/cockroachdb/cockroach/pkg/workload/ttllogger"
+	_ "github.com/cockroachdb/cockroach/pkg/workload/vecann"
 	_ "github.com/cockroachdb/cockroach/pkg/workload/ycsb"
 )

pkg/ccl/workloadccl/allccl/all_test.go

@@ -139,10 +139,14 @@ func TestAllRegisteredSetup(t *testing.T) {
 				t.Fatal(err)
 			}
 		case `interleavedpartitioned`:
-			// This require a specific node locality setup
+			// This require a specific node locality setup.
 			continue
 		case `ttlbench`:
 			continue
+		case `vecann`:
+			// This requires downloading from a GCP bucket and storing in the
+			// machine's ~/.cache directory.
+			continue
 		}
 
 		t.Run(meta.Name, func(t *testing.T) {

pkg/cmd/vecbench/BUILD.bazel

@@ -7,7 +7,6 @@ go_library(
         "main.go",
         "mem_provider.go",
         "percentile_estimator.go",
-        "progress_writer.go",
         "sql_provider.go",
         "vector_provider.go",
     ],
@@ -18,18 +17,18 @@ go_library(
         "//pkg/sql/vecindex/cspann/memstore",
         "//pkg/sql/vecindex/cspann/quantize",
         "//pkg/util/httputil",
+        "//pkg/util/humanizeutil",
         "//pkg/util/stop",
         "//pkg/util/syncutil",
         "//pkg/util/timeutil",
         "//pkg/util/vector",
+        "//pkg/workload/vecann",
         "@com_github_cockroachdb_crlib//crtime",
         "@com_github_cockroachdb_errors//:errors",
         "@com_github_cockroachdb_errors//oserror",
         "@com_github_guptarohit_asciigraph//:asciigraph",
-        "@com_github_jackc_pgx_v5//:pgx",
         "@com_github_jackc_pgx_v5//pgconn",
         "@com_github_jackc_pgx_v5//pgxpool",
-        "@com_google_cloud_go_storage//:storage",
         "@org_golang_x_term//:term",
     ],
 )