Merge pull request #139 from weaviate/update-to-handle-125-changes

Update to handle core `v1.25` changes

Author: tsmith023

.github/workflows/main.yaml

@@ -8,7 +8,7 @@ on:
 
 
 env:
-  WEAVIATE_VERSION: 1.24.1
+  WEAVIATE_VERSION: 1.25.1
 
 jobs:
   checks:
@@ -27,15 +27,23 @@ jobs:
   tests:
     needs: checks
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        versions: [
+          { node: "18.x", weaviate: $WEAVIATE_VERSION},
+          { node: "20.x", weaviate: $WEAVIATE_VERSION},
+          { node: "22.x", weaviate: $WEAVIATE_VERSION}
+        ]
     steps:
     - uses: actions/checkout@v3
     - uses: actions/setup-node@v3
       with:
-        node-version: '18.x'
+        node-version: ${{ matrix.versions.node }}
     - name: "Install dependencies"
       run: |
         npm install
-        ci/run_dependencies.sh ${{ env.WEAVIATE_VERSION }}
+        ci/run_dependencies.sh ${{ matrix.versions.weaviate }}
     - name: "Run tests with authentication tests"
       if: ${{ !github.event.pull_request.head.repo.fork }}
       env:
@@ -52,7 +60,7 @@ jobs:
         npm test
         npm run build
     - name: "Stop Weaviate"
-      run: ci/stop_dependencies.sh ${{ env.WEAVIATE_VERSION }}
+      run: ci/stop_dependencies.sh ${{ matrix.versions.weaviate }}
 
   publish:
     needs: tests

ci/docker-compose-cluster.yml

@@ -6,16 +6,22 @@ services:
     restart: on-failure:0
     ports:
       - "8087:8080"
+      - "50058:50051"
     environment:
-      CONTEXTIONARY_URL: contextionary:9999
       QUERY_DEFAULTS_LIMIT: 20
       AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'true'
       PERSISTENCE_DATA_PATH: "./weaviate-node-1"
-      DEFAULT_VECTORIZER_MODULE: text2vec-contextionary
-      ENABLE_MODULES: text2vec-contextionary
+      CLUSTER_HOSTNAME: "node1"
       CLUSTER_GOSSIP_BIND_PORT: "7110"
       CLUSTER_DATA_BIND_PORT: "7111"
+      RAFT_PORT: '8300'
+      RAFT_INTERNAL_RPC_PORT: "8301"
+      RAFT_JOIN: "node1:8300,node2:8300,node3:8300"
+      RAFT_BOOTSTRAP_EXPECT: "3"
       DISABLE_TELEMETRY: 'true'
+      CONTEXTIONARY_URL: contextionary:9999
+      DEFAULT_VECTORIZER_MODULE: text2vec-contextionary
+      ENABLE_MODULES: text2vec-contextionary
 
   weaviate-node-2:
     init: true
@@ -29,19 +35,55 @@ services:
     image: semitechnologies/weaviate:${WEAVIATE_VERSION}
     ports:
       - 8088:8080
-      - 6061:6060
+      - "50059:50051"
     restart: on-failure:0
     environment:
-      CONTEXTIONARY_URL: contextionary:9999
       LOG_LEVEL: 'debug'
       QUERY_DEFAULTS_LIMIT: 20
       AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'true'
       PERSISTENCE_DATA_PATH: './weaviate-node-2'
+      CLUSTER_HOSTNAME: 'node2'
+      CLUSTER_GOSSIP_BIND_PORT: '7110'
+      CLUSTER_DATA_BIND_PORT: '7111'
+      CLUSTER_JOIN: 'weaviate-node-1:7110'
+      RAFT_PORT: '8300'
+      RAFT_INTERNAL_RPC_PORT: "8301"
+      RAFT_JOIN: "node1:8300,node2:8300,node3:8300"
+      RAFT_BOOTSTRAP_EXPECT: "3"
+      DISABLE_TELEMETRY: 'true'
+      CONTEXTIONARY_URL: contextionary:9999
       DEFAULT_VECTORIZER_MODULE: text2vec-contextionary
       ENABLE_MODULES: text2vec-contextionary
-      CLUSTER_HOSTNAME: 'node2'
-      CLUSTER_GOSSIP_BIND_PORT: '7112'
-      CLUSTER_DATA_BIND_PORT: '7113'
+
+  weaviate-node-3:
+    init: true
+    command:
+      - --host
+      - 0.0.0.0
+      - --port
+      - '8080'
+      - --scheme
+      - http
+    image: semitechnologies/weaviate:${WEAVIATE_VERSION}
+    ports:
+      - 8089:8080
+      - "50060:50051"
+    restart: on-failure:0
+    environment:
+      LOG_LEVEL: 'debug'
+      QUERY_DEFAULTS_LIMIT: 20
+      AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'true'
+      PERSISTENCE_DATA_PATH: './weaviate-node-2'
+      CLUSTER_HOSTNAME: 'node3'
+      CLUSTER_GOSSIP_BIND_PORT: '7110'
+      CLUSTER_DATA_BIND_PORT: '7111'
       CLUSTER_JOIN: 'weaviate-node-1:7110'
+      RAFT_PORT: '8300'
+      RAFT_INTERNAL_RPC_PORT: "8301"
+      RAFT_JOIN: "node1:8300,node2:8300,node3:8300"
+      RAFT_BOOTSTRAP_EXPECT: "3"
       DISABLE_TELEMETRY: 'true'
-...
+      CONTEXTIONARY_URL: contextionary:9999
+      DEFAULT_VECTORIZER_MODULE: text2vec-contextionary
+      ENABLE_MODULES: text2vec-contextionary
+...

src/connection/httpClient.ts

@@ -149,8 +149,8 @@ const makeCheckStatus = (expectResponseBody: boolean) => (res: Response) => {
 };
 
 const handleHeadResponse = (expectResponseBody: boolean) => (res: Response) => {
-  if (res.status == 204 || res.status == 404) {
-    return res.status == 204;
+  if (res.status == 200 || res.status == 204 || res.status == 404) {
+    return res.status == 200 || res.status == 204;
   }
   return makeCheckStatus(expectResponseBody)(res);
 };

src/graphql/getter.test.ts

@@ -1289,6 +1289,21 @@ describe('bm25 valid searchers', () => {
 
     expect(mockClient.query).toHaveBeenCalledWith(expectedQuery);
   });
+
+  test('query and groupby', () => {
+    const expectedQuery = `{Get{Person(bm25:{query:"accountant"},groupBy:{path:["employer"],groups:2,objectsPerGroup:3}){name}}}`;
+
+    new Getter(mockClient)
+      .withClassName('Person')
+      .withFields('name')
+      .withBm25({
+        query: 'accountant',
+      })
+      .withGroupBy({ path: ['employer'], groups: 2, objectsPerGroup: 3 })
+      .do();
+
+    expect(mockClient.query).toHaveBeenCalledWith(expectedQuery);
+  });
 });
 
 describe('hybrid valid searchers', () => {
@@ -1383,6 +1398,50 @@ describe('hybrid valid searchers', () => {
 
     expect(mockClient.query).toHaveBeenCalledWith(expectedQuery);
   });
+
+  test('query and groupby', () => {
+    const expectedQuery = `{Get{Person(hybrid:{query:"accountant"},groupBy:{path:["employer"],groups:2,objectsPerGroup:3}){name}}}`;
+
+    new Getter(mockClient)
+      .withClassName('Person')
+      .withFields('name')
+      .withHybrid({
+        query: 'accountant',
+      })
+      .withGroupBy({ path: ['employer'], groups: 2, objectsPerGroup: 3 })
+      .do();
+
+    expect(mockClient.query).toHaveBeenCalledWith(expectedQuery);
+  });
+
+  test('query and subsearches', () => {
+    const subQuery = `searches:[{nearVector:{vector:[1,2,3],certainty:0.8,targetVectors:["employer"]}},{nearText:{concepts:["accountant"],distance:0.3,moveTo:{concepts:["foo"],objects:[{id:"uuid"}],force:0.8}}}]`;
+    const expectedQuery = `{Get{Person(hybrid:{query:"accountant",${subQuery}}){name}}}`;
+
+    new Getter(mockClient)
+      .withClassName('Person')
+      .withFields('name')
+      .withHybrid({
+        query: 'accountant',
+        searches: [
+          { nearVector: { certainty: 0.8, targetVectors: ['employer'], vector: [1, 2, 3] } },
+          {
+            nearText: {
+              concepts: ['accountant'],
+              distance: 0.3,
+              moveTo: {
+                concepts: ['foo'],
+                objects: [{ id: 'uuid' }],
+                force: 0.8,
+              },
+            },
+          },
+        ],
+      })
+      .do();
+
+    expect(mockClient.query).toHaveBeenCalledWith(expectedQuery);
+  });
 });
 
 describe('generative search', () => {

src/graphql/hybrid.ts

@@ -1,24 +1,92 @@
+import { Move, parseMove } from './nearText';
+
 export interface HybridArgs {
   alpha?: number;
   query: string;
   vector?: number[];
   properties?: string[];
   targetVectors?: string[];
   fusionType?: FusionType;
+  searches?: HybridSubSearch[];
+}
+
+export interface NearTextSubSearch {
+  concepts: string[];
+  certainty?: number;
+  distance?: number;
+  moveAwayFrom?: Move;
+  moveTo?: Move;
+}
+
+export interface NearVectorSubSearch {
+  vector: number[];
+  certainty?: number;
+  distance?: number;
+  targetVectors?: string[];
+}
+
+export interface HybridSubSearch {
+  nearText?: NearTextSubSearch;
+  nearVector?: NearVectorSubSearch;
 }
 
 export enum FusionType {
   rankedFusion = 'rankedFusion',
   relativeScoreFusion = 'relativeScoreFusion',
 }
 
+class GraphQLHybridSubSearch {
+  private nearText?: NearTextSubSearch;
+  private nearVector?: NearVectorSubSearch;
+
+  constructor(args: HybridSubSearch) {
+    this.nearText = args.nearText;
+    this.nearVector = args.nearVector;
+  }
+
+  toString(): string {
+    let outer: string[] = [];
+    if (this.nearText !== undefined) {
+      let inner = [`concepts:${JSON.stringify(this.nearText.concepts)}`];
+      if (this.nearText.certainty) {
+        inner = [...inner, `certainty:${this.nearText.certainty}`];
+      }
+      if (this.nearText.distance) {
+        inner = [...inner, `distance:${this.nearText.distance}`];
+      }
+      if (this.nearText.moveTo) {
+        inner = [...inner, parseMove('moveTo', this.nearText.moveTo)];
+      }
+      if (this.nearText.moveAwayFrom) {
+        inner = [...inner, parseMove('moveAwayFrom', this.nearText.moveAwayFrom)];
+      }
+      outer = [...outer, `nearText:{${inner.join(',')}}`];
+    }
+    if (this.nearVector !== undefined) {
+      let inner = [`vector:${JSON.stringify(this.nearVector.vector)}`];
+      if (this.nearVector.certainty) {
+        inner = [...inner, `certainty:${this.nearVector.certainty}`];
+      }
+      if (this.nearVector.distance) {
+        inner = [...inner, `distance:${this.nearVector.distance}`];
+      }
+      if (this.nearVector.targetVectors && this.nearVector.targetVectors.length > 0) {
+        inner = [...inner, `targetVectors:${JSON.stringify(this.nearVector.targetVectors)}`];
+      }
+      outer = [...outer, `nearVector:{${inner.join(',')}}`];
+    }
+    return `{${outer.join(',')}}`;
+  }
+}
+
 export default class GraphQLHybrid {
   private alpha?: number;
   private query: string;
   private vector?: number[];
   private properties?: string[];
   private targetVectors?: string[];
   private fusionType?: FusionType;
+  private searches?: GraphQLHybridSubSearch[];
 
   constructor(args: HybridArgs) {
     this.alpha = args.alpha;
@@ -27,6 +95,7 @@ export default class GraphQLHybrid {
     this.properties = args.properties;
     this.targetVectors = args.targetVectors;
     this.fusionType = args.fusionType;
+    this.searches = args.searches?.map((search) => new GraphQLHybridSubSearch(search));
   }
 
   toString() {
@@ -52,6 +121,10 @@ export default class GraphQLHybrid {
       args = [...args, `fusionType:${this.fusionType}`];
     }
 
+    if (this.searches !== undefined) {
+      args = [...args, `searches:[${this.searches.map((search) => search.toString()).join(',')}]`];
+    }
+
     return `{${args.join(',')}}`;
   }
 }