fix wiki embeddings (#23)

* fix wiki embeddings

The old ones were deleted.

Requires us to switch to 1024d vectors.

* dataset cache

Author: nvb

cmd/nightly/datasource_cohere.go

@@ -84,7 +84,7 @@ func run(ctx context.Context, shutdown context.CancelFunc) error {
 	// for queries and small upserts).
 	executor := &TemplateExecutor{
 		nextId:  0,
-		vectors: RandomVectorSource(768),
+		vectors: RandomVectorSource(1024),
 		msmarco: &MSMarcoSource{},
 	}
 
@@ -142,7 +142,7 @@ func run(ctx context.Context, shutdown context.CancelFunc) error {
 	if err != nil {
 		return fmt.Errorf("failed to setup namespaces: %w", err)
 	}
-	executor.vectors = RandomVectorSource(768)
+	executor.vectors = RandomVectorSource(1024)
 
 	// Wait until the largest namespace has been fully indexed,
 	// i.e. we just dumped in a huge amount of documents
@@ -345,7 +345,7 @@ func setupNamespaces(
 
 	defer func() {
 		executor.lock.Lock()
-		executor.vectors = RandomVectorSource(768)
+		executor.vectors = RandomVectorSource(1024)
 		executor.lock.Unlock()
 	}()
 

data.go

@@ -1,6 +1,6 @@
 {
     "id": {{ id }},
-    "vector": {{ vector 768 }},
+    "vector": {{ vector 1024 }},
     "attributes": {
         "attr1": "{{ string_with_cardinality 1 10 }}",
         "attr2": "{{ string 32 }}",

main.go

@@ -1,4 +1,4 @@
 {
     "id": {{ id }},
-    "vector": {{ vector 768 }}
+    "vector": {{ vector 1024 }}
 }

templates/document_attrs.json.tmpl

@@ -1,5 +1,5 @@
-# name=ANN (10M, 768dims) - No filter, strongly consistent
+# name=ANN (10M, 1024dims) - No filter, strongly consistent
 {
     "top_k": 10,
-    "rank_by": ["vector", "ANN", {{ vector 768 }}]
+    "rank_by": ["vector", "ANN", {{ vector 1024 }}]
 }

templates/document_default.json.tmpl

@@ -1,6 +1,6 @@
-# name=ANN (10M, 768dims) - No filter, cold
+# name=ANN (10M, 1024dims) - No filter, cold
 {
     "top_k": 10,
-    "rank_by": ["vector", "ANN", {{ vector 768 }}],
+    "rank_by": ["vector", "ANN", {{ vector 1024 }}],
     "disable_cache": true
 }

templates/nightly/10m_768dim/ann.tmpl

@@ -1,7 +1,7 @@
-# name=ANN (10M, 768dims) - No filter, eventually consistent
+# name=ANN (10M, 1024dims) - No filter, eventually consistent
 {
     "top_k": 10,
-    "rank_by": ["vector", "ANN", {{ vector 768 }}],
+    "rank_by": ["vector", "ANN", {{ vector 1024 }}],
     "consistency": {
         "level": "eventual"
     }

templates/nightly/10m_768dim/ann_cold.tmpl

@@ -3,7 +3,7 @@
 {{define "row"}}
 {
     "id": {{ id }},
-    "vector": {{ vector 768 }},
+    "vector": {{ vector 1024 }},
     "text": "{{ paragraph }}"
 }
 {{end}}

templates/nightly/10m_768dim/ann_eventual_consistency.tmpl

@@ -1,6 +1,6 @@
 {
     "top_k": 10,
-    "rank_by": ["vector", "ANN", {{ vector 768 }}],
+    "rank_by": ["vector", "ANN", {{ vector 1024 }}],
     "filter": ["attr1", "Eq", "a"],
     "include_attributes": [
         "attr2",