Merge #144533

144533: sql/vecindex: support operator class and index type syntax r=mw5h a=DrewKimball

#### sql/vecindex: ability to specify distance functions for index creation

This commit adds support for specifying the operator class for a vector
index (e.g. `vector_l2_ops`, `vector_cosine_ops` etc.). Currently, only
`vector_l2_ops` is supported. It is also the default option, which means
that users can omit the operator class in index definitions.

Fixes #143110

Release note (sql change): It is now possible to specify the operator
class for a vector index, although only `vector_l2_ops` is supported for
now. In addition, `vector_l2_ops` is the default, so it is possible to
omit the operator class from an index definition.

#### sql/vecindex: make hnsw alias for cspann for index creation

This commit adds `hnsw` to the accepted list of index types when a
vector index is defined using the `USING` syntax. However, we will
silently provide a `cspann` index instead. This change is made in
anticipation of third-party tools that want to use `cspann`.

Fixes #143109

Release note (sql change): When creating a vector index via the `USING`
syntax, is is now possible to specify `hnsw` as the index type, although
a `cspann` vector index will still be provided. The goal of this change
is to increase compatibility with third-party tools.

Co-authored-by: Drew Kimball <[email protected]>

Author: craig[bot]

pkg/ccl/logictestccl/testdata/logic_test/partitioning_index

@@ -206,11 +206,11 @@ SHOW CREATE TABLE vector
             b INT8 NULL,
             v VECTOR(3) NULL,
             CONSTRAINT vector_pkey PRIMARY KEY (a ASC),
-            VECTOR INDEX vector_b_v_idx (b ASC, v) PARTITION BY LIST (b) (
+            VECTOR INDEX vector_b_v_idx (b ASC, v vector_l2_ops) PARTITION BY LIST (b) (
               PARTITION p1 VALUES IN ((1)),
               PARTITION pu VALUES IN ((NULL))
             ),
-            VECTOR INDEX vector_idx (a ASC, b ASC, v) PARTITION BY LIST (a, b) (
+            VECTOR INDEX vector_idx (a ASC, b ASC, v vector_l2_ops) PARTITION BY LIST (a, b) (
               PARTITION p1 VALUES IN ((1, 1), (2, 2))
             )
           )

pkg/cmd/roachtest/testdata/pg_regress/btree_index.diffs

@@ -86,13 +86,13 @@ diff -U3 --label=/mnt/data1/postgres/src/test/regress/expected/btree_index.out -
  -- "lost".
  --
  CREATE INDEX bt_i4_index ON bt_i4_heap USING btree (seqno int4_ops);
-+ERROR:  operator classes are only allowed for the last column of an inverted index
++ERROR:  operator classes are only allowed for the last column of an inverted or vector index
  CREATE INDEX bt_name_index ON bt_name_heap USING btree (seqno name_ops);
-+ERROR:  operator classes are only allowed for the last column of an inverted index
++ERROR:  operator classes are only allowed for the last column of an inverted or vector index
  CREATE INDEX bt_txt_index ON bt_txt_heap USING btree (seqno text_ops);
-+ERROR:  operator classes are only allowed for the last column of an inverted index
++ERROR:  operator classes are only allowed for the last column of an inverted or vector index
  CREATE INDEX bt_f8_index ON bt_f8_heap USING btree (seqno float8_ops);
-+ERROR:  operator classes are only allowed for the last column of an inverted index
++ERROR:  operator classes are only allowed for the last column of an inverted or vector index
  --
  -- test retrieval of min/max keys for each index
  --

pkg/cmd/roachtest/testdata/pg_regress/create_index.diffs

@@ -10,10 +10,10 @@ diff -U3 --label=/mnt/data1/postgres/src/test/regress/expected/create_index.out
  -- BTREE
  --
  CREATE INDEX onek_unique1 ON onek USING btree(unique1 int4_ops);
-+ERROR:  operator classes are only allowed for the last column of an inverted index
++ERROR:  operator classes are only allowed for the last column of an inverted or vector index
  CREATE INDEX IF NOT EXISTS onek_unique1 ON onek USING btree(unique1 int4_ops);
 -NOTICE:  relation "onek_unique1" already exists, skipping
-+ERROR:  operator classes are only allowed for the last column of an inverted index
++ERROR:  operator classes are only allowed for the last column of an inverted or vector index
  CREATE INDEX IF NOT EXISTS ON onek USING btree(unique1 int4_ops);
 -ERROR:  syntax error at or near "ON"
 -LINE 1: CREATE INDEX IF NOT EXISTS ON onek USING btree(unique1 int4_...
@@ -24,24 +24,24 @@ diff -U3 --label=/mnt/data1/postgres/src/test/regress/expected/create_index.out
 +                              ^
 +HINT:  try \h CREATE INDEX
  CREATE INDEX onek_unique2 ON onek USING btree(unique2 int4_ops);
-+ERROR:  operator classes are only allowed for the last column of an inverted index
++ERROR:  operator classes are only allowed for the last column of an inverted or vector index
  CREATE INDEX onek_hundred ON onek USING btree(hundred int4_ops);
-+ERROR:  operator classes are only allowed for the last column of an inverted index
++ERROR:  operator classes are only allowed for the last column of an inverted or vector index
  CREATE INDEX onek_stringu1 ON onek USING btree(stringu1 name_ops);
-+ERROR:  operator classes are only allowed for the last column of an inverted index
++ERROR:  operator classes are only allowed for the last column of an inverted or vector index
  CREATE INDEX tenk1_unique1 ON tenk1 USING btree(unique1 int4_ops);
-+ERROR:  operator classes are only allowed for the last column of an inverted index
++ERROR:  operator classes are only allowed for the last column of an inverted or vector index
  CREATE INDEX tenk1_unique2 ON tenk1 USING btree(unique2 int4_ops);
-+ERROR:  operator classes are only allowed for the last column of an inverted index
++ERROR:  operator classes are only allowed for the last column of an inverted or vector index
  CREATE INDEX tenk1_hundred ON tenk1 USING btree(hundred int4_ops);
-+ERROR:  operator classes are only allowed for the last column of an inverted index
++ERROR:  operator classes are only allowed for the last column of an inverted or vector index
  CREATE INDEX tenk1_thous_tenthous ON tenk1 (thousand, tenthous);
  CREATE INDEX tenk2_unique1 ON tenk2 USING btree(unique1 int4_ops);
-+ERROR:  operator classes are only allowed for the last column of an inverted index
++ERROR:  operator classes are only allowed for the last column of an inverted or vector index
  CREATE INDEX tenk2_unique2 ON tenk2 USING btree(unique2 int4_ops);
-+ERROR:  operator classes are only allowed for the last column of an inverted index
++ERROR:  operator classes are only allowed for the last column of an inverted or vector index
  CREATE INDEX tenk2_hundred ON tenk2 USING btree(hundred int4_ops);
-+ERROR:  operator classes are only allowed for the last column of an inverted index
++ERROR:  operator classes are only allowed for the last column of an inverted or vector index
  CREATE INDEX rix ON road USING btree (name text_ops);
 +ERROR:  relation "road" does not exist
  CREATE INDEX iix ON ihighway USING btree (name text_ops);
@@ -61,13 +61,13 @@ diff -U3 --label=/mnt/data1/postgres/src/test/regress/expected/create_index.out
  --
  CREATE INDEX onek2_u1_prtl ON onek2 USING btree(unique1 int4_ops)
  	where unique1 < 20 or unique1 > 980;
-+ERROR:  operator classes are only allowed for the last column of an inverted index
++ERROR:  operator classes are only allowed for the last column of an inverted or vector index
  CREATE INDEX onek2_u2_prtl ON onek2 USING btree(unique2 int4_ops)
  	where stringu1 < 'B';
-+ERROR:  operator classes are only allowed for the last column of an inverted index
++ERROR:  operator classes are only allowed for the last column of an inverted or vector index
  CREATE INDEX onek2_stu1_prtl ON onek2 USING btree(stringu1 name_ops)
  	where onek2.stringu1 >= 'J' and onek2.stringu1 < 'K';
-+ERROR:  operator classes are only allowed for the last column of an inverted index
++ERROR:  operator classes are only allowed for the last column of an inverted or vector index
  --
  -- GiST (rtree-equivalent opclasses only)
  --
@@ -2086,7 +2086,7 @@ diff -U3 --label=/mnt/data1/postgres/src/test/regress/expected/create_index.out
  DROP TABLE func_index_heap;
  CREATE TABLE func_index_heap (f1 text, f2 text);
  CREATE UNIQUE INDEX func_index_index on func_index_heap ((f1 || f2) text_ops);
-+ERROR:  operator classes are only allowed for the last column of an inverted index
++ERROR:  operator classes are only allowed for the last column of an inverted or vector index
  INSERT INTO func_index_heap VALUES('ABC','DEF');
  INSERT INTO func_index_heap VALUES('AB','CDEFG');
  INSERT INTO func_index_heap VALUES('QWE','RTY');

pkg/cmd/roachtest/testdata/pg_regress/insert_conflict.diffs

@@ -5,7 +5,7 @@ diff -U3 --label=/mnt/data1/postgres/src/test/regress/expected/insert_conflict.o
  -- named collations
  --
  create unique index op_index_key on insertconflicttest(key, fruit text_pattern_ops);
-+ERROR:  operator classes are only allowed for the last column of an inverted index
++ERROR:  operator classes are only allowed for the last column of an inverted or vector index
  create unique index collation_index_key on insertconflicttest(key, fruit collate "C");
 +ERROR:  at or near "collate": syntax error
 +DETAIL:  source SQL:

pkg/cmd/roachtest/testdata/pg_regress/plpgsql.diffs

@@ -5,75 +5,75 @@ diff -U3 --label=/mnt/data1/postgres/src/test/regress/expected/plpgsql.out --lab
      comment	text
  );
  create unique index Room_rno on Room using btree (roomno bpchar_ops);
-+ERROR:  operator classes are only allowed for the last column of an inverted index
++ERROR:  operator classes are only allowed for the last column of an inverted or vector index
  create table WSlot (
      slotname	char(20),
      roomno	char(8),
 @@ -32,11 +33,13 @@
      backlink	char(20)
  );
  create unique index WSlot_name on WSlot using btree (slotname bpchar_ops);
-+ERROR:  operator classes are only allowed for the last column of an inverted index
++ERROR:  operator classes are only allowed for the last column of an inverted or vector index
  create table PField (
      name	text,
      comment	text
  );
  create unique index PField_name on PField using btree (name text_ops);
-+ERROR:  operator classes are only allowed for the last column of an inverted index
++ERROR:  operator classes are only allowed for the last column of an inverted or vector index
  create table PSlot (
      slotname	char(20),
      pfname	text,
 @@ -44,6 +47,7 @@
      backlink	char(20)
  );
  create unique index PSlot_name on PSlot using btree (slotname bpchar_ops);
-+ERROR:  operator classes are only allowed for the last column of an inverted index
++ERROR:  operator classes are only allowed for the last column of an inverted or vector index
  create table PLine (
      slotname	char(20),
      phonenumber	char(20),
 @@ -51,12 +55,14 @@
      backlink	char(20)
  );
  create unique index PLine_name on PLine using btree (slotname bpchar_ops);
-+ERROR:  operator classes are only allowed for the last column of an inverted index
++ERROR:  operator classes are only allowed for the last column of an inverted or vector index
  create table Hub (
      name	char(14),
      comment	text,
      nslots	integer
  );
  create unique index Hub_name on Hub using btree (name bpchar_ops);
-+ERROR:  operator classes are only allowed for the last column of an inverted index
++ERROR:  operator classes are only allowed for the last column of an inverted or vector index
  create table HSlot (
      slotname	char(20),
      hubname	char(14),
 @@ -64,12 +70,15 @@
      slotlink	char(20)
  );
  create unique index HSlot_name on HSlot using btree (slotname bpchar_ops);
-+ERROR:  operator classes are only allowed for the last column of an inverted index
++ERROR:  operator classes are only allowed for the last column of an inverted or vector index
  create index HSlot_hubname on HSlot using btree (hubname bpchar_ops);
-+ERROR:  operator classes are only allowed for the last column of an inverted index
++ERROR:  operator classes are only allowed for the last column of an inverted or vector index
  create table System (
      name	text,
      comment	text
  );
  create unique index System_name on System using btree (name text_ops);
-+ERROR:  operator classes are only allowed for the last column of an inverted index
++ERROR:  operator classes are only allowed for the last column of an inverted or vector index
  create table IFace (
      slotname	char(20),
      sysname	text,
 @@ -77,12 +86,14 @@
      slotlink	char(20)
  );
  create unique index IFace_name on IFace using btree (slotname bpchar_ops);
-+ERROR:  operator classes are only allowed for the last column of an inverted index
++ERROR:  operator classes are only allowed for the last column of an inverted or vector index
  create table PHone (
      slotname	char(20),
      comment	text,
      slotlink	char(20)
  );
  create unique index PHone_name on PHone using btree (slotname bpchar_ops);
-+ERROR:  operator classes are only allowed for the last column of an inverted index
++ERROR:  operator classes are only allowed for the last column of an inverted or vector index
  -- ************************************************************
  -- *
  -- * Trigger procedures and functions for the patchfield

pkg/sql/catalog/catformat/index.go

@@ -248,13 +248,19 @@ func FormatIndexElements(
 		} else {
 			f.FormatNameP(&index.KeyColumnNames[i])
 		}
-		// TODO(drewk): we might need to print something like "vector_l2_ops" for
-		// vector indexes.
-		if index.Type == idxtype.INVERTED &&
-			col.GetID() == index.InvertedColumnID() && len(index.InvertedColumnKinds) > 0 {
-			switch index.InvertedColumnKinds[0] {
-			case catpb.InvertedIndexColumnKind_TRIGRAM:
-				f.WriteString(" gin_trgm_ops")
+		switch index.Type {
+		case idxtype.INVERTED:
+			if col.GetID() == index.InvertedColumnID() && len(index.InvertedColumnKinds) > 0 {
+				switch index.InvertedColumnKinds[0] {
+				case catpb.InvertedIndexColumnKind_TRIGRAM:
+					f.WriteString(" gin_trgm_ops")
+				}
+			}
+		case idxtype.VECTOR:
+			// TODO(#144016): once more distance functions are supported, store the
+			// operator on the index and use it here.
+			if col.GetID() == index.VectorColumnID() {
+				f.WriteString(" vector_l2_ops")
 			}
 		}
 		// The last column of an inverted or vector index cannot have a DESC

pkg/sql/catalog/catformat/index_test.go

@@ -277,16 +277,16 @@ func TestIndexForDisplay(t *testing.T) {
 			tableName:   descpb.AnonymousTable,
 			partition:   "",
 			displayMode: IndexDisplayDefOnly,
-			expected:    "VECTOR INDEX baz (a)",
-			pgExpected:  "INDEX baz USING cspann (a)",
+			expected:    "VECTOR INDEX baz (a vector_l2_ops)",
+			pgExpected:  "INDEX baz USING cspann (a vector_l2_ops)",
 		},
 		{
 			index:       vectorIndex,
 			tableName:   tableName,
 			partition:   "",
 			displayMode: IndexDisplayShowCreate,
-			expected:    "CREATE VECTOR INDEX baz ON foo.public.bar (a)",
-			pgExpected:  "CREATE INDEX baz ON foo.public.bar USING cspann (a)",
+			expected:    "CREATE VECTOR INDEX baz ON foo.public.bar (a vector_l2_ops)",
+			pgExpected:  "CREATE INDEX baz ON foo.public.bar USING cspann (a vector_l2_ops)",
 		},
 	}
 

pkg/sql/create_index.go

@@ -256,6 +256,17 @@ func makeIndexDescriptor(
 		}
 
 		vecCol := columns[len(columns)-1]
+		switch vecCol.OpClass {
+		case "vector_l2_ops", "":
+			// vector_l2_ops is the default operator class. This allows users to omit
+			// the operator class in index definitions.
+		case "vector_l1_ops", "vector_ip_ops", "vector_cosine_ops",
+			"bit_hamming_ops", "bit_jaccard_ops":
+			return nil, unimplemented.NewWithIssuef(144016,
+				"operator class %v is not supported", vecCol.OpClass)
+		default:
+			return nil, newUndefinedOpclassError(vecCol.OpClass)
+		}
 		column, err := catalog.MustFindColumnByTreeName(tableDesc, vecCol.Column)
 		if err != nil {
 			return nil, err
@@ -372,7 +383,7 @@ func checkIndexColumns(
 		}
 		if colDef.OpClass != "" && (i < len(columns)-1 || !indexType.SupportsOpClass()) {
 			return pgerror.New(pgcode.DatatypeMismatch,
-				"operator classes are only allowed for the last column of an inverted index")
+				"operator classes are only allowed for the last column of an inverted or vector index")
 		}
 	}
 	for i, colName := range storing {

pkg/sql/logictest/testdata/logic_test/create_index

@@ -422,12 +422,12 @@ CREATE TABLE opclasses (a INT PRIMARY KEY, b TEXT, c JSON)
 
 # Make sure that we don't permit creating indexes with opclasses that aren't
 # inverted.
-statement error pgcode 42804 operator classes are only allowed for the last column of an inverted index
+statement error pgcode 42804 operator classes are only allowed for the last column of an inverted or vector index
 CREATE INDEX ON opclasses(b blah_ops)
 
 # Make sure that we don't permit creating indexes with opclasses that aren't
 # inverted, even if the type is invertible.
-statement error pgcode 42804 operator classes are only allowed for the last column of an inverted index
+statement error pgcode 42804 operator classes are only allowed for the last column of an inverted or vector index
 CREATE INDEX ON opclasses(c blah_ops)
 
 # Make sure that we don't permit creating indexes with opclasses that don't exist

pkg/sql/logictest/testdata/logic_test/create_table

@@ -219,7 +219,7 @@ like_indexes  CREATE TABLE public.like_indexes (
                 UNIQUE INDEX foo (b DESC, c ASC),
                 INDEX like_table_c_idx (c ASC) STORING (j),
                 INVERTED INDEX like_table_j_idx (j),
-                VECTOR INDEX like_table_v_idx (v)
+                VECTOR INDEX like_table_v_idx (v vector_l2_ops)
               )
 
 # INCLUDING GENERATED adds "generated columns", aka stored columns.
@@ -280,7 +280,7 @@ like_all  CREATE TABLE public.like_all (
             UNIQUE INDEX foo (b DESC, c ASC),
             INDEX like_table_c_idx (c ASC) STORING (j),
             INVERTED INDEX like_table_j_idx (j),
-            VECTOR INDEX like_table_v_idx (v),
+            VECTOR INDEX like_table_v_idx (v vector_l2_ops),
             CONSTRAINT check_a CHECK (a > 3:::INT8),
             CONSTRAINT unique_k UNIQUE WITHOUT INDEX (k),
             CONSTRAINT unique_h UNIQUE WITHOUT INDEX (h),
@@ -308,7 +308,7 @@ like_mixed  CREATE TABLE public.like_mixed (
               UNIQUE INDEX foo (b DESC, c ASC),
               INDEX like_table_c_idx (c ASC) STORING (j),
               INVERTED INDEX like_table_j_idx (j),
-              VECTOR INDEX like_table_v_idx (v)
+              VECTOR INDEX like_table_v_idx (v vector_l2_ops)
             )
 
 statement ok