v6: Update to latest Weaviate versions

.github/workflows/test.yaml

@@ -4,33 +4,98 @@ on:
     branches:
       - main
   pull_request:
+concurrency:
+  group: tests-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  IMG2VEC: cr.weaviate.io/semitechnologies/img2vec-pytorch:resnet50
+  MINIO: minio/minio
+  MODEL2VEC: cr.weaviate.io/semitechnologies/model2vec-inference:minishlab-potion-base-4M
+  DOCKER_IMAGES_TAR: docker-images.tar
 
 jobs:
-  tests:
-    name: Tests
+  docker-cache:
+    name: Cache shared Docker images
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v4
-      with:
-        ref: ${{ github.sha }}
     - name: Login to Docker Hub
       if: ${{ !github.event.pull_request.head.repo.fork }}
-      uses: docker/login-action@v2
+      uses: docker/login-action@v3
+      with:
+        username: ${{ secrets.DOCKER_USERNAME }}
+        password: ${{ secrets.DOCKER_PASSWORD }}
+    - id: cache-check
+      uses: actions/cache/restore@v4
+      env:
+        DOCKER_CACHE_KEY: docker-images-${{ env.IMG2VEC }}-${{ env.MINIO }}-${{ env.MODEL2VEC }}
+      with:
+        path: ${{ env.DOCKER_IMAGES_TAR }}
+        key: ${{ env.DOCKER_CACHE_KEY }}
+        lookup-only: true  # Only check if cache exists, don't download
+    - name: Pull images
+      if: steps.cache-check.outputs.cache-hit != 'true'
+      run: |
+        docker pull $IMG2VEC
+        docker pull $MINIO
+        # docker pull $MODEL2VEC
+        docker save $IMG2VEC $MINIO -o $DOCKER_IMAGES_TAR
+    - name: Cache images
+      if: steps.cache-check.outputs.cache-hit != 'true'
+      uses: actions/cache/save@v4
+      env:
+        DOCKER_CACHE_KEY: docker-images-${{ env.IMG2VEC }}-${{ env.MINIO }}-${{ env.MODEL2VEC }}
+      with:
+        path: ${{ env.DOCKER_IMAGES_TAR }}
+        key: ${{ env.DOCKER_CACHE_KEY }}
+
+  maven-cache:
+      name: Cache Maven dependencies
+      runs-on: ubuntu-latest
+      steps:
+        - uses: actions/checkout@v4
+        - uses: actions/setup-java@v4
+          with:
+            distribution: 'zulu'
+            java-version: '17'
+            cache: 'maven'
+        - run: mvn dependency:go-offline
+
+  test:
+    name: Test
+    runs-on: ubuntu-latest
+    needs: [ docker-cache, maven-cache]
+    strategy:
+      fail-fast: false
+      matrix:
+        WEAVIATE_VERSION: ["1.32.16", "1.33.4", "1.34.0"]
+    steps:
+    - uses: actions/checkout@v4
+
+    - uses: actions/cache/restore@v4
+      env:
+        DOCKER_CACHE_KEY: docker-images-${{ env.IMG2VEC }}-${{ env.MINIO }}-${{ env.MODEL2VEC }}
       with:
-        username: ${{secrets.DOCKER_USERNAME}}
-        password: ${{secrets.DOCKER_PASSWORD}}
-    - name: Setup JDK
-      uses: actions/setup-java@v4
+        path: ${{ env.DOCKER_IMAGES_TAR }}
+        key: ${{ env.DOCKER_CACHE_KEY }}
+    - name: Load Docker images
+      run: |
+        if [ -f $DOCKER_IMAGES_TAR ]; then
+          docker load -i $DOCKER_IMAGES_TAR
+        fi
+    - uses: actions/setup-java@v4
+      name: Setup JDK
       with:
         distribution: 'zulu'
         java-version: '17'
-    - name: Run Build
-      run: mvn -DskipTests clean package
-    - name: Run Tests
+        cache: 'maven'
+
+    - name: Run Tests (v${{ matrix.WEAVIATE_VERSION }})
       env:
         OKTA_DUMMY_CI_PW: ${{ secrets.OKTA_DUMMY_CI_PW }}
         WCS_DUMMY_CI_PW: ${{ secrets.WCS_DUMMY_CI_PW }}
         OKTA_CLIENT_SECRET: ${{ secrets.OKTA_CLIENT_SECRET }}
         AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
         OPENAI_APIKEY: ${{ secrets.OPENAI_APIKEY }}
-      run: mvn clean test
+        WEAVIATE_VERSION: ${{ matrix.WEAVIATE_VERSION }}
+      run: mvn verify -Dgpg.skip

.gitignore

@@ -11,3 +11,5 @@
 target/
 # maven-lombok-plugin
 .factorypath
+# Surefire statistics for optimized execution time
+.surefire-*

pom.xml

@@ -52,7 +52,6 @@
     <maven.compiler.source>17</maven.compiler.source>
     <maven.compiler.target>17</maven.compiler.target>
     <maven.compiler.release>17</maven.compiler.release>
-    <lombok.version>1.18.42</lombok.version>
     <gson.version>2.13.2</gson.version>
     <httpclient.version>5.5.1</httpclient.version>
     <lang3.version>3.19.0</lang3.version>
@@ -112,11 +111,6 @@
       <artifactId>httpclient5</artifactId>
       <version>${httpclient.version}</version>
     </dependency>
-    <dependency>
-      <groupId>org.projectlombok</groupId>
-      <artifactId>lombok</artifactId>
-      <version>${lombok.version}</version>
-    </dependency>
     <dependency>
       <groupId>com.google.code.gson</groupId>
       <artifactId>gson</artifactId>
@@ -229,72 +223,17 @@
         </plugin>
         <plugin>
           <artifactId>maven-surefire-plugin</artifactId>
-          <version>2.22.2</version>
+          <version>3.5.4</version>
           <configuration>
             <trimStackTrace>false</trimStackTrace>
-            <argLine>
-              <!--
-                Gson (used for JSON serialization) utilizes reflection and needs to be able to access private fields of
-                Java core classes. For that to work correctly after Java 9, we need to open up some of the core Java
-                classes to reflection.
-                @see: https://www.oracle.com/corporate/features/understanding-java-9-modules.html
-              -->
-              --add-opens=java.base/java.lang=ALL-UNNAMED
-              </argLine>
-              <properties>
-                <property>
-                  <name>listener</name>
-                  <value>io.weaviate.containers.TestListener</value>
-                </property>
-              </properties>
+            <parallel>classes</parallel>
+            <forkCount>4</forkCount>
+            <reuseForks>true</reuseForks>
+            <threadCount>1</threadCount>
+            <perCoreThreadCount>true</perCoreThreadCount>
+            <runOrder>balanced</runOrder>
           </configuration>
         </plugin>
-        <plugin>
-          <groupId>org.projectlombok</groupId>
-          <artifactId>lombok-maven-plugin</artifactId>
-          <version>1.18.20.0</version>
-          <executions>
-            <execution>
-              <id>delombok-sources</id>
-              <phase>generate-sources</phase>
-              <goals>
-                <goal>delombok</goal>
-              </goals>
-              <configuration>
-                <sourceDirectory>src/main/java</sourceDirectory>
-                <outputDirectory>${project.build.directory}/delombok</outputDirectory>
-                <addOutputDirectory>false</addOutputDirectory>
-                <encoding>UTF-8</encoding>
-              </configuration>
-            </execution>
-          </executions>
-          <dependencies>
-            <dependency>
-              <groupId>org.projectlombok</groupId>
-              <artifactId>lombok</artifactId>
-              <version>${lombok.version}</version>
-            </dependency>
-          </dependencies>
-        </plugin>
-        <plugin>
-          <groupId>org.apache.maven.plugins</groupId>
-          <artifactId>maven-antrun-plugin</artifactId>
-          <version>3.0.0</version>
-          <executions>
-            <execution>
-              <id>generate-delomboked-sources-jar</id>
-              <phase>package</phase>
-              <goals>
-                <goal>run</goal>
-              </goals>
-              <configuration>
-                <target>
-                  <jar destfile="${project.build.directory}/${project.build.finalName}-sources.jar" basedir="${project.build.directory}/delombok"/>
-                </target>
-              </configuration>
-            </execution>
-          </executions>
-        </plugin>
         <plugin>
           <groupId>org.codehaus.mojo</groupId>
           <artifactId>versions-maven-plugin</artifactId>
@@ -317,22 +256,6 @@
           <artifactId>build-helper-maven-plugin</artifactId>
           <version>3.2.0</version>
           <executions>
-            <execution>
-              <id>attach-delomboked-sources-jar</id>
-              <phase>package</phase>
-              <goals>
-                <goal>attach-artifact</goal>
-              </goals>
-              <configuration>
-                <artifacts>
-                  <artifact>
-                    <file>${project.build.directory}/${project.build.finalName}-sources.jar</file>
-                    <type>jar</type>
-                    <classifier>sources</classifier>
-                  </artifact>
-                </artifacts>
-              </configuration>
-            </execution>
             <execution>
               <id>add-test-source</id>
               <phase>generate-test-sources</phase>
@@ -384,7 +307,6 @@
             </execution>
           </executions>
           <configuration>
-            <sourcepath>${project.build.directory}/delombok</sourcepath>
             <!--Disable warnings about missing docstrings.-->
             <doclint>all,-missing</doclint>
           </configuration>
@@ -512,10 +434,6 @@
       </plugins>
     </pluginManagement>
     <plugins>
-      <plugin>
-        <groupId>org.projectlombok</groupId>
-        <artifactId>lombok-maven-plugin</artifactId>
-      </plugin>
       <plugin>
         <groupId>org.xolstice.maven.plugins</groupId>
         <artifactId>protobuf-maven-plugin</artifactId>

src/it/java/io/weaviate/ConcurrentTest.java

@@ -10,9 +10,13 @@
 
 import org.apache.commons.lang3.RandomStringUtils;
 import org.assertj.core.api.Assertions;
+import org.assertj.core.api.Assumptions;
 import org.junit.Rule;
 import org.junit.rules.TestName;
 
+import io.weaviate.client6.v1.internal.VersionSupport.SemanticVersion;
+import io.weaviate.containers.Weaviate;
+
 /**
  * ConcurrentTest is the base class for integration tests, which provides
  * utility methods to uniqualize collections and objects created in the
@@ -110,4 +114,22 @@ public static void eventually(Callable<Boolean> cond, int intervalMillis, int ti
       throw new RuntimeException(ex);
     }
   }
+
+  /**
+   * Skip the test if the version that the {@link Weaviate}
+   * container is running is older than the required one.
+   */
+  public static void requireAtLeast(Weaviate.Version required) {
+    var actual = SemanticVersion.of(Weaviate.VERSION);
+    Assumptions.assumeThat(actual)
+        .as("requires at least %s, but running %s", required.semver, actual)
+        .isGreaterThanOrEqualTo(required.semver);
+  }
+
+  public static void requireAtLeast(Weaviate.Version required, Runnable r) {
+    var actual = SemanticVersion.of(Weaviate.VERSION);
+    if (actual.compareTo(required.semver) >= 0) {
+      r.run();
+    }
+  }
 }

src/it/java/io/weaviate/containers/Container.java

@@ -18,18 +18,6 @@ public class Container {
   public static final Img2VecNeural IMG2VEC_NEURAL = Img2VecNeural.createDefault();
   public static final MinIo MINIO = MinIo.createDefault();
 
-  /**
-   * Stop all shared Testcontainers created in {@link #startAll}.
-   * <p>
-   * Testcontainer's Ryuk will reap any dangling containers after the tests
-   * finish. However, since {@link Weaviate} instances also hold a
-   * {@link WeaviateClient}, we want to stop them proactively to
-   * close client connections.
-   */
-  static void stopAll() {
-    WEAVIATE.stop();
-  }
-
   public static ContainerGroup compose(Weaviate weaviate, GenericContainer<?>... containers) {
     return new ContainerGroup(weaviate, containers);
   }

src/it/java/io/weaviate/containers/Weaviate.java

@@ -16,18 +16,40 @@
 import org.testcontainers.lifecycle.Startable;
 import org.testcontainers.weaviate.WeaviateContainer;
 
+import io.weaviate.ConcurrentTest;
 import io.weaviate.client6.v1.api.Config;
 import io.weaviate.client6.v1.api.WeaviateClient;
 import io.weaviate.client6.v1.internal.ObjectBuilder;
+import io.weaviate.client6.v1.internal.VersionSupport.SemanticVersion;
 
 public class Weaviate extends WeaviateContainer {
-  public static final String VERSION = "1.33.0";
   public static final String DOCKER_IMAGE = "semitechnologies/weaviate";
+  public static final String LATEST_VERSION = "1.34.0";
+  public static final String VERSION;
+
+  static {
+    VERSION = System.getenv().getOrDefault("WEAVIATE_VERSION", LATEST_VERSION);
+  }
   public static String OIDC_ISSUER = "https://auth.wcs.api.weaviate.io/auth/realms/SeMI";
 
   private volatile SharedClient clientInstance;
   private final String containerName;
 
+  public enum Version {
+    V132(1, 32),
+    V133(1, 33);
+
+    public final SemanticVersion semver;
+
+    private Version(int major, int minor) {
+      this.semver = new SemanticVersion(major, minor);
+    }
+
+    public void orSkip() {
+      ConcurrentTest.requireAtLeast(this);
+    }
+  }
+
   /**
    * By default, testcontainer's name is only available after calling
    * {@link #start}.
@@ -71,6 +93,22 @@ public WeaviateClient getClient() {
     return clientInstance;
   }
 
+  /**
+   * Get client that is not shared with other tests / callers.
+   * The returned client is not wrapped in an instance of {@link SharedClient},
+   * so it can be auto-closed by the try-with-resources statement when it exists.
+   */
+  public WeaviateClient getBareClient() {
+    if (!isRunning()) {
+      start();
+    }
+    try {
+      return new WeaviateClient(Config.of(defaultConfigFn()));
+    } catch (Exception e) {
+      throw new RuntimeException("create WeaviateClient for Weaviate container", e);
+    }
+  }
+
   /**
    * Create a new instance of WeaviateClient connected to this container.
    * Prefer using {@link #getClient} unless your test needs the initialization