cspann: fix bug in CalculateTruth

Fix bug in CalculateTruth, which was calling vecpb.MeasureDistance with
un-normalized vectors for Cosine distance. This returns incorrect results.

Epic: CRDB-42943
Release note: None

Author: andy-kimball

pkg/sql/vecindex/cspann/testdata/search-embeddings.ddt

@@ -395,20 +395,20 @@ CV stats:
 
 recall topk=10 beam-size=4 samples=50
 ----
-20.80% recall@10
+30.00% recall@10
 45 leaf vectors, 74 vectors, 31 full vectors, 7 partitions
 
 recall topk=10 beam-size=8 samples=50
 ----
-28.40% recall@10
+43.40% recall@10
 88 leaf vectors, 139 vectors, 47 full vectors, 13 partitions
 
 recall topk=10 beam-size=16 samples=50
 ----
-34.60% recall@10
+62.80% recall@10
 175 leaf vectors, 265 vectors, 66 full vectors, 25 partitions
 
 recall topk=10 beam-size=32 samples=50
 ----
-38.60% recall@10
+81.80% recall@10
 348 leaf vectors, 447 vectors, 89 full vectors, 42 partitions

pkg/sql/vecindex/cspann/testutils/testutils.go

@@ -117,10 +117,22 @@ func CalculateTruth[T comparable](
 	dataVectors vector.Set,
 	dataKeys []T,
 ) []T {
+	var queryNorm float32
+	if distMetric == vecpb.CosineDistance {
+		// MeasureDistance assumes input vectors are normalized.
+		queryNorm = num32.Norm(queryVector)
+	}
 	distances := make([]float32, dataVectors.Count)
 	offsets := make([]int, dataVectors.Count)
 	for i := range dataVectors.Count {
-		distances[i] = vecpb.MeasureDistance(distMetric, queryVector, dataVectors.At(i))
+		data := dataVectors.At(i)
+		if distMetric == vecpb.CosineDistance {
+			// MeasureDistance assumes input vectors are normalized, so adjust the
+			// result.
+			distances[i] = 1 - num32.Dot(queryVector, data)/(queryNorm*num32.Norm(data))
+		} else {
+			distances[i] = vecpb.MeasureDistance(distMetric, queryVector, data)
+		}
 		offsets[i] = i
 	}
 	sort.SliceStable(offsets, func(i int, j int) bool {