Merge pull request #1454 from weaviate/dev/1.28

Dev/1.28

Author: dirkkul

.github/workflows/main.yaml

@@ -14,12 +14,10 @@ on:
   pull_request:
 
 env:
-  WEAVIATE_123: 1.23.16
-  WEAVIATE_124: 1.24.26
   WEAVIATE_125: 1.25.24
   WEAVIATE_126: 1.26.8
-  WEAVIATE_127: stable-v1.27-372397e
-
+  WEAVIATE_127: 1.27.1
+  WEAVIATE_128: 1.28.0-rc.0-09ea6b2
 jobs:
   lint-and-format:
     name: Run Linter and Formatter
@@ -114,79 +112,21 @@ jobs:
           name: coverage-report-integration-embedded
           path: coverage-integration-embedded.xml
 
-  integration-tests-v3:
-    name: Run Integration Tests v3
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        versions: [
-          { py: "3.9", weaviate: $WEAVIATE_126},
-          { py: "3.10", weaviate: $WEAVIATE_126},
-          { py: "3.11", weaviate: $WEAVIATE_123},
-          { py: "3.11", weaviate: $WEAVIATE_124},
-          { py: "3.11", weaviate: $WEAVIATE_125},
-          { py: "3.11", weaviate: $WEAVIATE_126},
-          { py: "3.12", weaviate: $WEAVIATE_127}
-        ]
-        optional_dependencies: [false]
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          fetch-tags: true
-      - uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.versions.py }}
-          cache: 'pip' # caching pip dependencies
-      - name: Login to Docker Hub
-        uses: docker/login-action@v3
-        if: ${{ !github.event.pull_request.head.repo.fork && github.triggering_actor != 'dependabot[bot]' }}
-        with:
-          username: ${{secrets.DOCKER_USERNAME}}
-          password: ${{secrets.DOCKER_PASSWORD}}
-      - run: |
-          pip install -r requirements-devel.txt
-          pip install .
-      - name: free space
-        run: sudo rm -rf /usr/local/lib/android
-      - name: start weaviate
-        run: /bin/bash ci/start_weaviate.sh ${{ matrix.versions.weaviate }}
-      - name: Run integration tests with auth secrets
-        if: ${{ !github.event.pull_request.head.repo.fork }}
-        env:
-          AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
-          OKTA_CLIENT_SECRET: ${{ secrets.OKTA_CLIENT_SECRET }}
-          WCS_DUMMY_CI_PW: ${{ secrets.WCS_DUMMY_CI_PW }}
-          OKTA_DUMMY_CI_PW: ${{ secrets.OKTA_DUMMY_CI_PW }}
-#          OPENAI_APIKEY: ${{ secrets.OPENAI_APIKEY }}  disabled until we have a working key
-        run: pytest -v --cov --cov-report=term-missing --cov=weaviate --cov-report xml:coverage-integration-v3.xml integration_v3
-      - name: Run integration tests without auth secrets (for forks)
-        if: ${{ github.event.pull_request.head.repo.fork }}
-        run: pytest -v --cov --cov-report=term-missing --cov=weaviate --cov-report xml:coverage-integration-v3.xml integration_v3
-      - name: Archive code coverage results
-        if: matrix.versions.py == '3.10' && (github.ref_name != 'main')
-        uses: actions/upload-artifact@v4
-        with:
-          name: coverage-report-integration-v3
-          path: coverage-integration-v3.xml
-      - name: stop weaviate
-        run: /bin/bash ci/stop_weaviate.sh ${{ matrix.versions.weaviate }}
-
   integration-tests:
     name: Run Integration Tests
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
         versions: [
-          { py: "3.9", weaviate: $WEAVIATE_126},
-          { py: "3.10", weaviate: $WEAVIATE_126},
-          { py: "3.11", weaviate: $WEAVIATE_123},
-          { py: "3.11", weaviate: $WEAVIATE_124},
+          { py: "3.9", weaviate: $WEAVIATE_128},
+          { py: "3.10", weaviate: $WEAVIATE_128},
           { py: "3.11", weaviate: $WEAVIATE_125},
           { py: "3.11", weaviate: $WEAVIATE_126},
-          { py: "3.12", weaviate: $WEAVIATE_127}
+          { py: "3.11", weaviate: $WEAVIATE_127},
+          { py: "3.11", weaviate: $WEAVIATE_128},
+          { py: "3.12", weaviate: $WEAVIATE_128},
+          { py: "3.13", weaviate: $WEAVIATE_128}
         ]
         optional_dependencies: [false]
     steps:
@@ -219,7 +159,7 @@ jobs:
           WCS_DUMMY_CI_PW: ${{ secrets.WCS_DUMMY_CI_PW }}
           OKTA_DUMMY_CI_PW: ${{ secrets.OKTA_DUMMY_CI_PW }}
 #          OPENAI_APIKEY: ${{ secrets.OPENAI_APIKEY }}  disabled until we have a working key
-        run: pytest -n auto -v --cov --cov-report=term-missing --cov=weaviate --cov-report xml:coverage-integration.xml integration
+        run: pytest -n auto --dist loadgroup -v --cov --cov-report=term-missing --cov=weaviate --cov-report xml:coverage-integration.xml integration
       - name: Run integration tests without auth secrets (for forks)
         if: ${{ github.event.pull_request.head.repo.fork }}
         run: pytest -v --cov --cov-report=term-missing --cov=weaviate --cov-report xml:coverage-integration.xml integration
@@ -239,10 +179,11 @@ jobs:
       fail-fast: false
       matrix:
         versions: [
-          { py: "3.9", weaviate: $WEAVIATE_125},
-          { py: "3.10", weaviate: $WEAVIATE_125},
-          { py: "3.11", weaviate: $WEAVIATE_125},
-          { py: "3.12", weaviate: $WEAVIATE_125}
+          { py: "3.9", weaviate: $WEAVIATE_128},
+          { py: "3.10", weaviate: $WEAVIATE_128},
+          { py: "3.11", weaviate: $WEAVIATE_128},
+          { py: "3.12", weaviate: $WEAVIATE_128},
+          { py: "3.13", weaviate: $WEAVIATE_128}
         ]
         optional_dependencies: [false]
     steps:
@@ -269,7 +210,7 @@ jobs:
         run: /bin/bash ci/stop_weaviate.sh ${{ matrix.versions.weaviate }}
 
   Codecov:
-    needs: [Unit-Tests, Integration-Tests, Integration-Tests-v3]
+    needs: [Unit-Tests, Integration-Tests]
     runs-on: ubuntu-latest
     if: github.ref_name != 'main'
     steps:
@@ -286,10 +227,6 @@ jobs:
         uses: actions/download-artifact@v4
         with:
           name: coverage-report-integration
-      - name: Download coverage integration v3
-        uses: actions/download-artifact@v4
-        with:
-          name: coverage-report-integration-v3
       - name: Download coverage integration embedded
         uses: actions/download-artifact@v4
         with:
@@ -298,7 +235,7 @@ jobs:
         uses: codecov/codecov-action@v4
         with:
           fail_ci_if_error: true
-          files: ./coverage-integration.xml, ./coverage-integration-v3.xml, ./coverage-integration-embedded.xml, ./coverage-test.xml, ./coverage-mock_tests.xml
+          files: ./coverage-integration.xml, ./coverage-integration-embedded.xml, ./coverage-test.xml, ./coverage-mock_tests.xml
           verbose: true
           token: ${{ secrets.CODECOV_TOKEN }}
 
@@ -334,11 +271,10 @@ jobs:
       fail-fast: false
       matrix:
         server: [
-          $WEAVIATE_123,
-          $WEAVIATE_124,
           $WEAVIATE_125,
           $WEAVIATE_126,
-          $WEAVIATE_127
+          $WEAVIATE_127,
+          $WEAVIATE_128
         ]
     steps:
       - name: Checkout
@@ -370,7 +306,7 @@ jobs:
           OKTA_CLIENT_SECRET: ${{ secrets.OKTA_CLIENT_SECRET }}
           WCS_DUMMY_CI_PW: ${{ secrets.WCS_DUMMY_CI_PW }}
           OKTA_DUMMY_CI_PW: ${{ secrets.OKTA_DUMMY_CI_PW }}
-        run: pytest -v -n auto integration
+        run: pytest -v -n auto --dist loadgroup integration
       - name: Run integration tests without auth secrets (for forks)
         if: ${{ github.event.pull_request.head.repo.fork }}
         run: pytest -v -n auto integration
@@ -417,4 +353,4 @@ jobs:
         with:
           generate_release_notes: true
           draft: true
-          files: dist/*.whl
+          files: dist/*.whl

.pre-commit-config.yaml

@@ -13,7 +13,7 @@ repos:
     - id: trailing-whitespace
 
 - repo: https://github.com/myint/autoflake
-  rev: v1.4
+  rev: v2.2.1 # autoflake v2.2.1 is the latest version that supports Python 3.12
   hooks:
       - id: autoflake
         args: [--in-place, --remove-all-unused-imports, --exclude=weaviate/proto/*]
@@ -36,13 +36,13 @@ repos:
       ]
       files: '^weaviate/collections'
 
--   repo: local
-    hooks:
+- repo: local
+  hooks:
     - id: mypy
       name: mypy
       entry: ./run-mypy.sh
       language: python
-      language_version: "3.11"
+      language_version: "3.12"
       # use require_serial so that script
       # is only called once per commit
       require_serial: true

ci/compose.sh

@@ -21,7 +21,7 @@ function compose_down_all {
 }
 
 function all_weaviate_ports {
-  echo "8090 8081 8087 8088 8089 8086 8082 8083 8075 8079 8085 8080" # in alphabetic order of appearance in docker-compose files
+  echo "8090 8081 8087 8088 8089 8086 8082 8083 8075 8079 8092 8085 8080" # in alphabetic order of appearance in docker-compose files
 }
 
 function wait(){

ci/docker-compose-rbac.yml

@@ -0,0 +1,30 @@
+---
+version: '3.4'
+services:
+  weaviate-rbac:
+    command:
+      - --host
+      - 0.0.0.0
+      - --port
+      - '8085'
+      - --scheme
+      - http
+      - --write-timeout=600s
+    image: semitechnologies/weaviate:${WEAVIATE_VERSION}
+    ports:
+      - 8092:8085
+      - "50063:50051"
+    restart: on-failure:0
+    environment:
+      ENABLE_MODULES: "generative-dummy,reranker-dummy"
+      AUTHENTICATION_APIKEY_ENABLED: "true"
+      AUTHENTICATION_APIKEY_ALLOWED_KEYS: "existing-key"
+      AUTHENTICATION_APIKEY_USERS: "existing-user"
+      AUTHORIZATION_ADMIN_USERS: "existing-user"
+      PERSISTENCE_DATA_PATH: "./data-weaviate-0"
+      CLUSTER_IN_LOCALHOST: "true"
+      CLUSTER_GOSSIP_BIND_PORT: "7100"
+      CLUSTER_DATA_BIND_PORT: "7101"
+      RAFT_BOOTSTRAP_EXPECT: "1"
+      AUTHORIZATION_ENABLE_RBAC: "true"
+...

integration/conftest.py

@@ -70,6 +70,7 @@ def __call__(
         self,
         headers: Optional[Dict[str, str]] = None,
         ports: Tuple[int, int] = (8080, 50051),
+        auth_credentials: Optional[weaviate.auth.AuthCredentials] = None,
     ) -> weaviate.WeaviateClient:
         """Typing for fixture."""
         ...
@@ -82,6 +83,7 @@ def client_factory() -> Generator[ClientFactory, None, None]:
     def _factory(
         headers: Optional[Dict[str, str]] = None,
         ports: Tuple[int, int] = (8080, 50051),
+        auth_credentials: Optional[weaviate.auth.AuthCredentials] = None,
     ) -> weaviate.WeaviateClient:
         nonlocal client_fixture
         if client_fixture is None:
@@ -90,6 +92,7 @@ def _factory(
                 grpc_port=ports[1],
                 port=ports[0],
                 additional_config=AdditionalConfig(timeout=(60, 120)),  # for image tests
+                auth_credentials=auth_credentials,
             )
         return client_fixture
 

integration_v3/test_backup_v4.py integration/test_backup_v4.pyintegration_v3/test_backup_v4.py renamed to integration/test_backup_v4.py

@@ -19,6 +19,8 @@
     BackupFailedException,
 )
 
+pytestmark = pytest.mark.xdist_group(name="backup")
+
 BACKEND = BackupStorage.FILESYSTEM
 
 PARAGRAPHS_IDS = [