Removing quantisation logic

Author: polterguy

backend/files/misc/sqlite/migrations/0059-removing-quantisations.sql

@@ -0,0 +1,5 @@
+
+/*
+ * Dropping old virtual VSS table(s).
+ */
+vector_quantize_cleanup('ml_training_snippets', 'embeddings');

backend/files/system/magic/ml_training_snippets-vss.get.hl

@@ -74,7 +74,7 @@ data.connect:[generic|magic]
    .sql:@"
 select vss.distance, ts.id, ts.created, ts.type, ts.pushed, ts.uri, ts.prompt, ts.completion, ts.filename, ts.cached, ts.meta, ts.embeddings as embedding_vss
    from ml_training_snippets ts 
-      inner join vector_quantize_scan_stream('ml_training_snippets', 'embeddings', vector_as_f32(@embedding)) as vss on vss.rowid = ts.id
+      inner join vector_full_scan_stream('ml_training_snippets', 'embeddings', vector_as_f32(@embedding)) as vss on vss.rowid = ts.id
    where 1"
 
    // Further parametrising invocation if we should.

backend/files/system/openai/magic.startup/magic.ai.get-context.hl

@@ -164,7 +164,7 @@ slots.create:magic.ai.get-context
       strings.concat
          .:@"
 select vss.distance, vss.rowid as id, ts.prompt, ts.completion, ts.uri, ts.cached
-	from vector_quantize_scan_stream('ml_training_snippets', 'embeddings', vector_as_f32(@embedding)) as vss
+	from vector_full_scan_stream('ml_training_snippets', 'embeddings', vector_as_f32(@embedding)) as vss
     	inner join ml_training_snippets ts on ts.id = vss.rowid
    where
       ts.type = @type and

backend/files/system/openai/magic.startup/magic.ai.search.hl

@@ -87,7 +87,7 @@ slots.create:magic.ai.search
       strings.concat
          .:@"
 select vss.distance, ts.prompt, ts.uri
-	from vector_quantize_scan_stream('ml_training_snippets', 'embeddings', vector_as_f32(@embedding)) as vss
+	from vector_full_scan_stream('ml_training_snippets', 'embeddings', vector_as_f32(@embedding)) as vss
     	inner join ml_training_snippets ts on ts.id = vss.rowid
    where
       ts.type = @type and "

backend/files/system/openai/magic.startup/magic.ai.vectorise.hl

@@ -228,25 +228,6 @@ select id, prompt, completion
             message:Done creating embeddings for your model
             type:success
 
-      // Signaling frontend about quantisation process.
-      sockets.signal:x:@.arguments/*/feedback-channel
-         args
-            message:Quantizing your type for faster retrieval
-            type:info
-
-      // Quantizing type.
-      data.connect:magic
-         data.execute:@"
-select vector_init('ml_training_snippets', 'embeddings', 'dimension=1536,type=FLOAT32,distance=cosine');
-select vector_quantize('ml_training_snippets', 'embeddings', 'max_memory=100MB');
-select vector_quantize_preload('ml_training_snippets', 'embeddings')"
-
-      // Signaling frontend about quantisation process.
-      sockets.signal:x:@.arguments/*/feedback-channel
-         args
-            message:Done quantizing your type
-            type:info
-
       // Signaling frontend.
       strings.concat
          .:"Done vectorising model '"

backend/files/system/openai/vectorise-snippet.post.hl

@@ -80,10 +80,6 @@ data.connect:[generic|magic]
    data.execute:update ml_training_snippets set embeddings = vector_as_f32(@embeddings) where id = @id
       @embeddings:x:@strings.concat
       @id:x:@.arguments/*/id
-   data.execute:@"
-select vector_init('ml_training_snippets', 'embeddings', 'dimension=1536,type=FLOAT32,distance=cosine');
-select vector_quantize('ml_training_snippets', 'embeddings', 'max_memory=100MB');
-select vector_quantize_preload('ml_training_snippets', 'embeddings')"
 
 // Returning success to caller.
 return