Fix parsing of files and buffers in nearMedia methods

Author: tsmith023

ci/docker-compose.yml

@@ -2,31 +2,43 @@
 version: '3.4'
 services:
   weaviate:
+    command:
+      - --host
+      - 0.0.0.0
+      - --port
+      - '8080'
+      - --scheme
+      - http
+      - --write-timeout=600s
     image: semitechnologies/weaviate:${WEAVIATE_VERSION}
-    restart: on-failure:0
     ports:
       - "8080:8080"
       - "50051:50051"
+    restart: on-failure:0
     environment:
       CONTEXTIONARY_URL: contextionary:9999
-      QUERY_DEFAULTS_LIMIT: 20
+      QUERY_DEFAULTS_LIMIT: 25
       AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'true'
-      PERSISTENCE_DATA_PATH: "./weaviate-data"
-      DEFAULT_VECTORIZER_MODULE: text2vec-contextionary
-      ENABLE_MODULES: text2vec-contextionary,backup-filesystem
+      PERSISTENCE_DATA_PATH: '/var/lib/weaviate'
+      DEFAULT_VECTORIZER_MODULE: 'text2vec-contextionary'
+      ENABLE_MODULES: text2vec-contextionary,backup-filesystem,img2vec-neural
       BACKUP_FILESYSTEM_PATH: "/tmp/backups"
       CLUSTER_GOSSIP_BIND_PORT: "7100"
       CLUSTER_DATA_BIND_PORT: "7101"
-      DISABLE_TELEMETRY: 'true'
+      CLUSTER_HOSTNAME: "node1"
       AUTOSCHEMA_ENABLED: 'false'
+      IMAGE_INFERENCE_API: "http://i2v-neural:8080"
+      DISABLE_TELEMETRY: 'true'
   contextionary:
-    image: semitechnologies/contextionary:en0.16.0-v1.2.1
-    ports:
-      - "9999:9999"
     environment:
       OCCURRENCE_WEIGHT_LINEAR_FACTOR: 0.75
       EXTENSIONS_STORAGE_MODE: weaviate
       EXTENSIONS_STORAGE_ORIGIN: http://weaviate:8080
       NEIGHBOR_OCCURRENCE_IGNORE_PERCENTILE: 5
       ENABLE_COMPOUND_SPLITTING: 'false'
+    image: semitechnologies/contextionary:en0.16.0-v1.2.0
+    ports:
+      - 9999:9999
+  i2v-neural:
+    image: semitechnologies/img2vec-pytorch:resnet50
 ...

src/collections/aggregate/index.ts

@@ -7,6 +7,7 @@ import { FilterValue } from '../filters/index.js';
 
 import { WeaviateQueryError } from '../../errors.js';
 import { Aggregator } from '../../graphql/index.js';
+import { toBase64FromMedia } from '../../index.js';
 import { Serialize } from '../serialize/index.js';
 
 export type AggregateBaseOptions<T, M> = {
@@ -345,12 +346,12 @@ class AggregateManager<T> implements Aggregate<T> {
     this.tenant = tenant;
 
     this.groupBy = {
-      nearImage: <M extends PropertiesMetrics<T> | undefined = undefined>(
-        image: string,
+      nearImage: async <M extends PropertiesMetrics<T> | undefined = undefined>(
+        image: string | Buffer,
         opts?: AggregateGroupByNearOptions<T, M>
       ): Promise<AggregateGroupByResult<T, M>[]> => {
         const builder = this.base(opts?.returnMetrics, opts?.filters, opts?.groupBy).withNearImage({
-          image: image,
+          image: await toBase64FromMedia(image),
           certainty: opts?.certainty,
           distance: opts?.distance,
           targetVectors: opts?.targetVector ? [opts.targetVector] : undefined,
@@ -488,12 +489,12 @@ class AggregateManager<T> implements Aggregate<T> {
     return new AggregateManager<T>(connection, name, dbVersionSupport, consistencyLevel, tenant);
   }
 
-  nearImage<M extends PropertiesMetrics<T>>(
-    image: string,
+  async nearImage<M extends PropertiesMetrics<T>>(
+    image: string | Buffer,
     opts?: AggregateNearOptions<T, M>
   ): Promise<AggregateResult<T, M>> {
     const builder = this.base(opts?.returnMetrics, opts?.filters).withNearImage({
-      image: image,
+      image: await toBase64FromMedia(image),
       certainty: opts?.certainty,
       distance: opts?.distance,
       targetVectors: opts?.targetVector ? [opts.targetVector] : undefined,
@@ -608,12 +609,12 @@ export interface Aggregate<T> {
    *
    * This method requires a vectorizer capable of handling base64-encoded images, e.g. `img2vec-neural`, `multi2vec-clip`, and `multi2vec-bind`.
    *
-   * @param {string} image The base64-encoded image to search for.
+   * @param {string | Buffer} image The image to search on. This can be a base64 string, a file path string, or a buffer.
    * @param {AggregateNearOptions<T, M>} [opts] The options for the request.
    * @returns {Promise<AggregateResult<T, M>[]>} The aggregated metrics for the objects returned by the vector search.
    */
   nearImage<M extends PropertiesMetrics<T>>(
-    image: string,
+    image: string | Buffer,
     opts?: AggregateNearOptions<T, M>
   ): Promise<AggregateResult<T, M>>;
   /**
@@ -678,12 +679,12 @@ export interface AggregateGroupBy<T> {
    *
    * This method requires a vectorizer capable of handling base64-encoded images, e.g. `img2vec-neural`, `multi2vec-clip`, and `multi2vec-bind`.
    *
-   * @param {string} image The base64-encoded image to search for.
+   * @param {string | Buffer} image The image to search on. This can be a base64 string, a file path string, or a buffer.
    * @param {AggregateGroupByNearOptions<T, M>} [opts] The options for the request.
    * @returns {Promise<AggregateGroupByResult<T, M>[]>} The aggregated metrics for the objects returned by the vector search.
    */
   nearImage<M extends PropertiesMetrics<T>>(
-    image: string,
+    image: string | Buffer,
     opts?: AggregateGroupByNearOptions<T, M>
   ): Promise<AggregateGroupByResult<T, M>[]>;
   /**

src/collections/generate/index.ts

@@ -4,6 +4,7 @@ import { ConsistencyLevel } from '../../data/index.js';
 import { DbVersionSupport } from '../../utils/dbVersion.js';
 
 import { WeaviateInvalidInputError, WeaviateUnsupportedFeatureError } from '../../errors.js';
+import { toBase64FromMedia } from '../../index.js';
 import { SearchReply } from '../../proto/v1/search_get.js';
 import { Deserialize } from '../deserialize/index.js';
 import {
@@ -160,26 +161,32 @@ class GenerateManager<T> implements Generate<T> {
   }
 
   public nearImage(
-    image: string,
+    image: string | Buffer,
     generate: GenerateOptions<T>,
     opts?: BaseNearOptions<T>
   ): Promise<GenerativeReturn<T>>;
   public nearImage(
-    image: string,
+    image: string | Buffer,
     generate: GenerateOptions<T>,
     opts: GroupByNearOptions<T>
   ): Promise<GenerativeGroupByReturn<T>>;
-  public nearImage(image: string, generate: GenerateOptions<T>, opts?: NearOptions<T>): GenerateReturn<T> {
+  public nearImage(
+    image: string | Buffer,
+    generate: GenerateOptions<T>,
+    opts?: NearOptions<T>
+  ): GenerateReturn<T> {
     return this.checkSupportForNamedVectors(opts)
       .then(() => this.connection.search(this.name, this.consistencyLevel, this.tenant))
       .then((search) =>
-        search.withNearImage({
-          ...Serialize.nearImage({ image, ...(opts ? opts : {}) }),
-          generative: Serialize.generative(generate),
-          groupBy: Serialize.isGroupBy<GroupByNearOptions<T>>(opts)
-            ? Serialize.groupBy(opts.groupBy)
-            : undefined,
-        })
+        toBase64FromMedia(image).then((image) =>
+          search.withNearImage({
+            ...Serialize.nearImage({ image, ...(opts ? opts : {}) }),
+            generative: Serialize.generative(generate),
+            groupBy: Serialize.isGroupBy<GroupByNearOptions<T>>(opts)
+              ? Serialize.groupBy(opts.groupBy)
+              : undefined,
+          })
+        )
       )
       .then((reply) => this.parseGroupByReply(opts, reply));
   }
@@ -268,19 +275,19 @@ class GenerateManager<T> implements Generate<T> {
   }
 
   public nearMedia(
-    media: string,
+    media: string | Buffer,
     type: NearMediaType,
     generate: GenerateOptions<T>,
     opts?: BaseNearOptions<T>
   ): Promise<GenerativeReturn<T>>;
   public nearMedia(
-    media: string,
+    media: string | Buffer,
     type: NearMediaType,
     generate: GenerateOptions<T>,
     opts: GroupByNearOptions<T>
   ): Promise<GenerativeGroupByReturn<T>>;
   public nearMedia(
-    media: string,
+    media: string | Buffer,
     type: NearMediaType,
     generate: GenerateOptions<T>,
     opts?: NearOptions<T>
@@ -295,46 +302,58 @@ class GenerateManager<T> implements Generate<T> {
           : undefined;
         switch (type) {
           case 'audio':
-            reply = search.withNearAudio({
-              ...Serialize.nearAudio({ audio: media, ...(opts ? opts : {}) }),
-              generative,
-              groupBy,
-            });
+            reply = toBase64FromMedia(media).then((media) =>
+              search.withNearAudio({
+                ...Serialize.nearAudio({ audio: media, ...(opts ? opts : {}) }),
+                generative,
+                groupBy,
+              })
+            );
             break;
           case 'depth':
-            reply = search.withNearDepth({
-              ...Serialize.nearDepth({ depth: media, ...(opts ? opts : {}) }),
-              generative,
-              groupBy,
-            });
+            reply = toBase64FromMedia(media).then((media) =>
+              search.withNearDepth({
+                ...Serialize.nearDepth({ depth: media, ...(opts ? opts : {}) }),
+                generative,
+                groupBy,
+              })
+            );
             break;
           case 'image':
-            reply = search.withNearImage({
-              ...Serialize.nearImage({ image: media, ...(opts ? opts : {}) }),
-              generative,
-              groupBy,
-            });
+            reply = toBase64FromMedia(media).then((media) =>
+              search.withNearImage({
+                ...Serialize.nearImage({ image: media, ...(opts ? opts : {}) }),
+                generative,
+                groupBy,
+              })
+            );
             break;
           case 'imu':
-            reply = search.withNearIMU({
-              ...Serialize.nearIMU({ imu: media, ...(opts ? opts : {}) }),
-              generative,
-              groupBy,
-            });
+            reply = toBase64FromMedia(media).then((media) =>
+              search.withNearIMU({
+                ...Serialize.nearIMU({ imu: media, ...(opts ? opts : {}) }),
+                generative,
+                groupBy,
+              })
+            );
             break;
           case 'thermal':
-            reply = search.withNearThermal({
-              ...Serialize.nearThermal({ thermal: media, ...(opts ? opts : {}) }),
-              generative,
-              groupBy,
-            });
+            reply = toBase64FromMedia(media).then((media) =>
+              search.withNearThermal({
+                ...Serialize.nearThermal({ thermal: media, ...(opts ? opts : {}) }),
+                generative,
+                groupBy,
+              })
+            );
             break;
           case 'video':
-            reply = search.withNearVideo({
-              ...Serialize.nearVideo({ video: media, ...(opts ? opts : {}) }),
-              generative,
-              groupBy,
-            });
+            reply = toBase64FromMedia(media).then((media) =>
+              search.withNearVideo({
+                ...Serialize.nearVideo({ video: media, ...(opts ? opts : {}) }),
+                generative,
+                groupBy,
+              })
+            );
             break;
           default:
             throw new WeaviateInvalidInputError(`Invalid media type: ${type}`);