Merge pull request #240 from weaviate/add-support-for-rbac

Add support for RBAC with tests, update CI images

Author: tsmith023

.github/workflows/main.yaml

@@ -8,9 +8,10 @@ on:
 
 env:
   WEAVIATE_124: 1.24.26
-  WEAVIATE_125: 1.25.25
-  WEAVIATE_126: 1.26.10
-  WEAVIATE_127: 1.27.3
+  WEAVIATE_125: 1.25.28
+  WEAVIATE_126: 1.26.13
+  WEAVIATE_127: 1.27.8
+  WEAVIATE_128: 1.28.1-77a2178
 
 jobs:
   checks:
@@ -37,9 +38,10 @@ jobs:
           { node: "22.x", weaviate: $WEAVIATE_124},
           { node: "22.x", weaviate: $WEAVIATE_125},
           { node: "22.x", weaviate: $WEAVIATE_126},
-          { node: "18.x", weaviate: $WEAVIATE_127},
-          { node: "20.x", weaviate: $WEAVIATE_127},
-          { node: "22.x", weaviate: $WEAVIATE_127}
+          { node: "22.x", weaviate: $WEAVIATE_127},
+          { node: "18.x", weaviate: $WEAVIATE_128},
+          { node: "20.x", weaviate: $WEAVIATE_128},
+          { node: "22.x", weaviate: $WEAVIATE_128}
         ]
     steps:
     - uses: actions/checkout@v3

ci/docker-compose-proxy.yml

@@ -26,16 +26,6 @@ services:
       AUTOSCHEMA_ENABLED: 'false'
       DISABLE_TELEMETRY: 'true'
       GRPC_PORT: 8021
-  contextionary:
-    environment:
-      OCCURRENCE_WEIGHT_LINEAR_FACTOR: 0.75
-      EXTENSIONS_STORAGE_MODE: weaviate
-      EXTENSIONS_STORAGE_ORIGIN: http://weaviate-proxy:8020
-      NEIGHBOR_OCCURRENCE_IGNORE_PERCENTILE: 5
-      ENABLE_COMPOUND_SPLITTING: 'false'
-    image: semitechnologies/contextionary:en0.16.0-v1.2.0
-    ports:
-      - 9999:9999
   proxy-http:
     image: envoyproxy/envoy:v1.29-latest
     command: envoy --config-path /etc/envoy/http.yaml

src/collections/config/types/index.ts

@@ -41,7 +41,10 @@ export type MultiTenancyConfig = {
   enabled: boolean;
 };
 
-export type ReplicationDeletionStrategy = 'DeleteOnConflict' | 'NoAutomatedResolution';
+export type ReplicationDeletionStrategy =
+  | 'DeleteOnConflict'
+  | 'NoAutomatedResolution'
+  | 'TimeBasedResolution';
 
 export type ReplicationConfig = {
   asyncEnabled: boolean;

src/collections/integration.test.ts

@@ -582,9 +582,7 @@ describe('Testing of the collections.create method', () => {
 
     expect(response.replication.asyncEnabled).toEqual(false);
     expect(response.replication.deletionStrategy).toEqual<ReplicationDeletionStrategy>(
-      (await cluster.getWeaviateVersion().then((ver) => ver.isLowerThan(1, 25, 0)))
-        ? 'NoAutomatedResolution'
-        : 'DeleteOnConflict'
+      'NoAutomatedResolution'
     );
     expect(response.replication.factor).toEqual(2);
 

src/connection/helpers.ts

@@ -157,13 +157,15 @@ export function connectToCustom(
 }
 
 function addWeaviateEmbeddingServiceHeaders(clusterURL: string, options?: ConnectToWeaviateCloudOptions) {
-  if (!isApiKey(options?.authCredentials)) {
+  const creds = options?.authCredentials;
+
+  if (!isApiKey(creds)) {
     return options?.headers;
   }
 
   return {
-    ...options.headers,
-    'X-Weaviate-Api-Key': mapApiKey(options.authCredentials).apiKey,
+    ...options?.headers,
+    'X-Weaviate-Api-Key': mapApiKey(creds).apiKey,
     'X-Weaviate-Cluster-Url': clusterURL,
   };
 }

src/connection/http.ts

@@ -4,8 +4,10 @@ import { Agent } from 'http';
 import OpenidConfigurationGetter from '../misc/openidConfigurationGetter.js';
 
 import {
+  WeaviateInsufficientPermissionsError,
   WeaviateInvalidInputError,
   WeaviateRequestTimeoutError,
+  WeaviateUnauthenticatedError,
   WeaviateUnexpectedStatusCodeError,
 } from '../errors.js';
 import {
@@ -155,11 +157,11 @@ export default class ConnectionREST {
     return this.http.head(path, payload);
   };
 
-  get = (path: string, expectReturnContent = true) => {
+  get = <T>(path: string, expectReturnContent = true) => {
     if (this.authEnabled) {
-      return this.login().then((token) => this.http.get(path, expectReturnContent, token));
+      return this.login().then((token) => this.http.get<T>(path, expectReturnContent, token));
     }
-    return this.http.get(path, expectReturnContent);
+    return this.http.get<T>(path, expectReturnContent);
   };
 
   login = async () => {
@@ -197,7 +199,7 @@ export interface HttpClient {
     expectReturnContent: boolean,
     bearerToken: string
   ) => Promise<T | undefined>;
-  get: (path: string, expectReturnContent?: boolean, bearerToken?: string) => any;
+  get: <T>(path: string, expectReturnContent?: boolean, bearerToken?: string) => Promise<T>;
   externalPost: (externalUrl: string, body: any, contentType: any) => any;
   getRaw: (path: string, bearerToken?: string) => any;
   delete: (path: string, payload: any, expectReturnContent?: boolean, bearerToken?: string) => any;
@@ -313,7 +315,7 @@ export const httpClient = (config: InternalConnectionParams): HttpClient => {
         handleHeadResponse<undefined>(false)
       );
     },
-    get: <T>(path: string, expectReturnContent = true, bearerToken = ''): Promise<T | undefined> => {
+    get: <T>(path: string, expectReturnContent = true, bearerToken = ''): Promise<T> => {
       const request = {
         method: 'GET',
         headers: {
@@ -323,7 +325,7 @@ export const httpClient = (config: InternalConnectionParams): HttpClient => {
       };
       addAuthHeaderIfNeeded(request, bearerToken);
       return fetchWithTimeout(url(path), config.timeout?.query || 30, request).then(
-        checkStatus<T>(expectReturnContent)
+        checkStatus<any>(expectReturnContent)
       );
     },
     getRaw: (path: string, bearerToken = '') => {
@@ -380,7 +382,13 @@ const checkStatus =
         } catch (e) {
           err = errText;
         }
-        return Promise.reject(new WeaviateUnexpectedStatusCodeError(res.status, err));
+        if (res.status === 401) {
+          return Promise.reject(new WeaviateUnauthenticatedError(err));
+        } else if (res.status === 403) {
+          return Promise.reject(new WeaviateInsufficientPermissionsError(403, err));
+        } else {
+          return Promise.reject(new WeaviateUnexpectedStatusCodeError(res.status, err));
+        }
       });
     }
     if (expectResponseBody) {

src/errors.ts

@@ -148,3 +148,20 @@ export class WeaviateRequestTimeoutError extends WeaviateError {
     super(`Weaviate request timed out with message: ${message}`);
   }
 }
+
+/**
+ * Is thrown if a request to Weaviate fails with a forbidden status code due to insufficient permissions.
+ */
+export class WeaviateInsufficientPermissionsError extends WeaviateError {
+  public code: number;
+  constructor(code: number, message: string) {
+    super(`Forbidden: ${message}`);
+    this.code = code;
+  }
+}
+
+export class WeaviateUnauthenticatedError extends WeaviateError {
+  constructor(message: string) {
+    super(`Unauthenticated: ${message}`);
+  }
+}

src/grpc/batcher.ts

@@ -1,12 +1,16 @@
-import { Metadata } from 'nice-grpc';
+import { Metadata, ServerError, Status } from 'nice-grpc';
 
 import { ConsistencyLevel } from '../data/index.js';
 
 import { BatchObject, BatchObjectsReply, BatchObjectsRequest } from '../proto/v1/batch.js';
 import { WeaviateClient } from '../proto/v1/weaviate.js';
 
 import { RetryOptions } from 'nice-grpc-client-middleware-retry';
-import { WeaviateBatchError, WeaviateDeleteManyError } from '../errors.js';
+import {
+  WeaviateBatchError,
+  WeaviateDeleteManyError,
+  WeaviateInsufficientPermissionsError,
+} from '../errors.js';
 import { Filters } from '../proto/v1/base.js';
 import { BatchDeleteReply, BatchDeleteRequest } from '../proto/v1/batch_delete.js';
 import Base from './base.js';
@@ -58,6 +62,9 @@ export default class Batcher extends Base implements Batch {
         }
       )
     ).catch((err) => {
+      if (err instanceof ServerError && err.code === Status.PERMISSION_DENIED) {
+        throw new WeaviateInsufficientPermissionsError(7, err.message);
+      }
       throw new WeaviateDeleteManyError(err.message);
     });
   }
@@ -77,6 +84,9 @@ export default class Batcher extends Base implements Batch {
           }
         )
         .catch((err) => {
+          if (err instanceof ServerError && err.code === Status.PERMISSION_DENIED) {
+            throw new WeaviateInsufficientPermissionsError(7, err.message);
+          }
           throw new WeaviateBatchError(err.message);
         })
     );

src/grpc/searcher.ts

@@ -1,6 +1,6 @@
 import { ConsistencyLevel } from '../data/index.js';
 
-import { Metadata } from 'nice-grpc';
+import { Metadata, ServerError, Status } from 'nice-grpc';
 import { Filters } from '../proto/v1/base.js';
 import {
   BM25,
@@ -9,8 +9,8 @@ import {
   MetadataRequest,
   NearAudioSearch,
   NearDepthSearch,
-  NearImageSearch,
   NearIMUSearch,
+  NearImageSearch,
   NearObject,
   NearTextSearch,
   NearThermalSearch,
@@ -25,7 +25,7 @@ import {
 import { WeaviateClient } from '../proto/v1/weaviate.js';
 
 import { RetryOptions } from 'nice-grpc-client-middleware-retry';
-import { WeaviateQueryError } from '../errors.js';
+import { WeaviateInsufficientPermissionsError, WeaviateQueryError } from '../errors.js';
 import { GenerativeSearch } from '../proto/v1/generative.js';
 import Base from './base.js';
 import { retryOptions } from './retry.js';
@@ -157,6 +157,9 @@ export default class Searcher extends Base implements Search {
           }
         )
         .catch((err) => {
+          if (err instanceof ServerError && err.code === Status.PERMISSION_DENIED) {
+            throw new WeaviateInsufficientPermissionsError(7, err.message);
+          }
           throw new WeaviateQueryError(err.message, 'gRPC');
         })
     );

src/grpc/tenantsManager.ts

@@ -1,5 +1,6 @@
-import { Metadata } from 'nice-grpc';
+import { Metadata, ServerError, Status } from 'nice-grpc';
 import { RetryOptions } from 'nice-grpc-client-middleware-retry';
+import { WeaviateDeleteManyError, WeaviateInsufficientPermissionsError } from '../errors.js';
 import { TenantsGetReply, TenantsGetRequest } from '../proto/v1/tenants.js';
 import { WeaviateClient } from '../proto/v1/weaviate.js';
 import Base from './base.js';
@@ -28,17 +29,24 @@ export default class TenantsManager extends Base implements Tenants {
 
   private call(message: TenantsGetRequest) {
     return this.sendWithTimeout((signal: AbortSignal) =>
-      this.connection.tenantsGet(
-        {
-          ...message,
-          collection: this.collection,
-        },
-        {
-          metadata: this.metadata,
-          signal,
-          ...retryOptions,
-        }
-      )
+      this.connection
+        .tenantsGet(
+          {
+            ...message,
+            collection: this.collection,
+          },
+          {
+            metadata: this.metadata,
+            signal,
+            ...retryOptions,
+          }
+        )
+        .catch((err) => {
+          if (err instanceof ServerError && err.code === Status.PERMISSION_DENIED) {
+            throw new WeaviateInsufficientPermissionsError(7, err.message);
+          }
+          throw new WeaviateDeleteManyError(err.message);
+        })
     );
   }
 }