HHH-19705 Add vector support for DB2

Author: beikov

docker_db.sh

@@ -4,9 +4,11 @@ if command -v docker > /dev/null; then
   CONTAINER_CLI=$(command -v docker)
   HEALTCHECK_PATH="{{.State.Health.Status}}"
   PRIVILEGED_CLI=""
+  IS_PODMAN=false
 else
   CONTAINER_CLI=$(command -v podman)
   HEALTCHECK_PATH="{{.State.Healthcheck.Status}}"
+  IS_PODMAN=true
   # Only use sudo for podman
   if command -v sudo > /dev/null; then
     PRIVILEGED_CLI="sudo"
@@ -15,6 +17,12 @@ else
   fi
 fi
 
+if [[ "$(uname -s)" == "Darwin" ]]; then
+  IS_OSX=true
+else
+  IS_OSX=false
+fi
+
 mysql() {
     mysql_9_4
 }
@@ -292,29 +300,93 @@ db2() {
 }
 
 db2_11_5() {
-    $PRIVILEGED_CLI $CONTAINER_CLI rm -f db2 || true
-    $PRIVILEGED_CLI $CONTAINER_CLI run --name db2 --privileged -e DB2INSTANCE=orm_test -e DB2INST1_PASSWORD=orm_test -e DBNAME=orm_test -e LICENSE=accept -e AUTOCONFIG=false -e ARCHIVE_LOGS=false -e TO_CREATE_SAMPLEDB=false -e REPODB=false -p 50000:50000 -d ${DB_IMAGE_DB2_11_5:-icr.io/db2_community/db2:11.5.9.0}
+    CONTAINER_OPTIONS=""
+    if [[ "$IS_OSX" == "true" ]]; then
+        # Thanks to Mohamed Asfour https://community.ibm.com/community/user/discussion/db2-luw-115xx-mac-m1-ready#bm017584d2-8d76-42a6-8f76-018dac8e78f2
+        # This SO post explains what goes wrong on OSX: https://stackoverflow.com/questions/70175677/ibmcom-db2-docker-image-fails-on-m1
+        # Also, use the $HOME directory as base directory to make volume mounts work on Colima on Mac
+        db2install=$HOME/db2install.sh
+        rm -f ${db2install} || true
+        cat <<'EOF' >${db2install}
+#!/bin/bash
+find /var/db2_setup -type f -not -path '*/\.*' -exec sed -i "s/su - db2inst1 -c '/su - db2inst1 -c '. .profile \&\& /g" {} +
+find /var/db2_setup -type f -not -path '*/\.*' -exec sed -i "s/su - db2inst1 -c \"/su - db2inst1 -c \". .profile \&\& /g" {} +
+find /var/db2_setup -type f -not -path '*/\.*' -exec sed -i "s/su - \${DB2INSTANCE?} -c '/su - \${DB2INSTANCE?} -c '. .profile \&\& /g" {} +
+find /var/db2_setup -type f -not -path '*/\.*' -exec sed -i "s/su - \${DB2INSTANCE?} -c \"/su - \${DB2INSTANCE?} -c \". .profile \&\& /g" {} +
+find /var/db2_setup -type f -not -path '*/\.*' -exec sed -i "s/su - \${instance?} -c '/su - \${instance?} -c '. .profile \&\& /g" {} +
+find /var/db2_setup -type f -not -path '*/\.*' -exec sed -i "s/su - \${instance?} -c \"/su - \${instance?} -c \". .profile \&\& /g" {} +
+find /var/db2_setup -type f -not -path '*/\.*' -exec sed -i "s/su - \${instance_name?} -c '/su - \${instance_name?} -c '. .profile \&\& /g" {} +
+find /var/db2_setup -type f -not -path '*/\.*' -exec sed -i "s/su - \${instance_name?} -c \"/su - \${instance_name?} -c \". .profile \&\& /g" {} +
+find /var/db2_setup -type f -not -path '*/\.*' -exec sed -i "s/su - db2inst1 -c \\\\\"/su - db2inst1 -c \\\". .profile \&\& /g" {} +
+. /var/db2_setup/lib/setup_db2_instance.sh
+EOF
+        chmod 777 ${db2install}
+        if [[ "$IS_PODMAN" == "true" ]]; then
+            CONTAINER_OPTIONS='-e IS_OSXFS=true -v '${db2install}':/db2install.sh --entrypoint=["/bin/bash","-c","/db2install.sh"]'
+            CONTAINER_ARGS=
+        else
+            CONTAINER_OPTIONS='-e IS_OSXFS=true -v '${db2install}':/db2install.sh --entrypoint=/bin/bash'
+            CONTAINER_ARGS=" -c /db2install.sh"
+        fi
+    fi
+    if [[ "$IS_PODMAN" == "false" ]]; then
+        export DOCKER_DEFAULT_PLATFORM=linux/amd64
+    fi
+    $CONTAINER_CLI rm -f db2 || true
+    $CONTAINER_CLI run --name db2 --privileged --platform=linux/amd64 -e DB2INSTANCE=orm_test -e DB2INST1_PASSWORD=orm_test -e DBNAME=orm_test -e LICENSE=accept -e AUTOCONFIG=false -e ARCHIVE_LOGS=false -e TO_CREATE_SAMPLEDB=false -e REPODB=false -p 50000:50000 $CONTAINER_OPTIONS -d ${DB_IMAGE_DB2_11_5:-icr.io/db2_community/db2:11.5.9.0} $CONTAINER_ARGS
     # Give the container some time to start
     OUTPUT=
     while [[ $OUTPUT != *"INSTANCE"* ]]; do
         echo "Waiting for DB2 to start..."
         sleep 10
-        OUTPUT=$($PRIVILEGED_CLI $CONTAINER_CLI logs db2 2>&1)
+        OUTPUT=$($CONTAINER_CLI logs db2 2>&1)
     done
-    $PRIVILEGED_CLI $CONTAINER_CLI exec -t db2 su - orm_test bash -c ". /database/config/orm_test/sqllib/db2profile; /database/config/orm_test/sqllib/bin/db2 'connect to orm_test'; /database/config/orm_test/sqllib/bin/db2 'CREATE USER TEMPORARY TABLESPACE usr_tbsp MANAGED BY AUTOMATIC STORAGE'"
+    $CONTAINER_CLI exec -t db2 su - orm_test bash -c ". /database/config/orm_test/sqllib/db2profile; /database/config/orm_test/sqllib/bin/db2 'connect to orm_test'; /database/config/orm_test/sqllib/bin/db2 'CREATE USER TEMPORARY TABLESPACE usr_tbsp MANAGED BY AUTOMATIC STORAGE'"
 }
 
 db2_12_1() {
-    $PRIVILEGED_CLI $CONTAINER_CLI rm -f db2 || true
-    $PRIVILEGED_CLI $CONTAINER_CLI run --name db2 --privileged -e DB2INSTANCE=orm_test -e DB2INST1_PASSWORD=orm_test -e DBNAME=orm_test -e LICENSE=accept -e AUTOCONFIG=false -e ARCHIVE_LOGS=false -e TO_CREATE_SAMPLEDB=false -e REPODB=false -p 50000:50000 -d ${DB_IMAGE_DB2_11_5:-icr.io/db2_community/db2:12.1.2.0}
+    CONTAINER_OPTIONS=""
+    if [[ "$IS_OSX" == "true" ]]; then
+        # Thanks to Mohamed Asfour https://community.ibm.com/community/user/discussion/db2-luw-115xx-mac-m1-ready#bm017584d2-8d76-42a6-8f76-018dac8e78f2
+        # This SO post explains what goes wrong on OSX: https://stackoverflow.com/questions/70175677/ibmcom-db2-docker-image-fails-on-m1
+        # Also, use the $HOME directory as base directory to make volume mounts work on Colima on Mac
+        db2install=$HOME/db2install.sh
+        rm -f ${db2install} || true
+        cat <<'EOF' >${db2install}
+#!/bin/bash
+find /var/db2_setup -type f -not -path '*/\.*' -exec sed -i "s/su - db2inst1 -c '/su - db2inst1 -c '. .profile \&\& /g" {} +
+find /var/db2_setup -type f -not -path '*/\.*' -exec sed -i "s/su - db2inst1 -c \"/su - db2inst1 -c \". .profile \&\& /g" {} +
+find /var/db2_setup -type f -not -path '*/\.*' -exec sed -i "s/su - \${DB2INSTANCE?} -c '/su - \${DB2INSTANCE?} -c '. .profile \&\& /g" {} +
+find /var/db2_setup -type f -not -path '*/\.*' -exec sed -i "s/su - \${DB2INSTANCE?} -c \"/su - \${DB2INSTANCE?} -c \". .profile \&\& /g" {} +
+find /var/db2_setup -type f -not -path '*/\.*' -exec sed -i "s/su - \${instance?} -c '/su - \${instance?} -c '. .profile \&\& /g" {} +
+find /var/db2_setup -type f -not -path '*/\.*' -exec sed -i "s/su - \${instance?} -c \"/su - \${instance?} -c \". .profile \&\& /g" {} +
+find /var/db2_setup -type f -not -path '*/\.*' -exec sed -i "s/su - \${instance_name?} -c '/su - \${instance_name?} -c '. .profile \&\& /g" {} +
+find /var/db2_setup -type f -not -path '*/\.*' -exec sed -i "s/su - \${instance_name?} -c \"/su - \${instance_name?} -c \". .profile \&\& /g" {} +
+find /var/db2_setup -type f -not -path '*/\.*' -exec sed -i "s/su - db2inst1 -c \\\\\"/su - db2inst1 -c \\\". .profile \&\& /g" {} +
+. /var/db2_setup/lib/setup_db2_instance.sh
+EOF
+        chmod 777 ${db2install}
+        if [[ "$IS_PODMAN" == "true" ]]; then
+            CONTAINER_OPTIONS='-e IS_OSXFS=true -v '${db2install}':/db2install.sh --entrypoint=["/bin/bash","-c","/db2install.sh"]'
+            CONTAINER_ARGS=
+        else
+            CONTAINER_OPTIONS='-e IS_OSXFS=true -v '${db2install}':/db2install.sh --entrypoint=/bin/bash'
+            CONTAINER_ARGS=" -c /db2install.sh"
+        fi
+    fi
+    if [[ "$IS_PODMAN" == "false" ]]; then
+        export DOCKER_DEFAULT_PLATFORM=linux/amd64
+    fi
+    $CONTAINER_CLI rm -f db2 || true
+    $CONTAINER_CLI run --name db2 --privileged --platform=linux/amd64 -e DB2INSTANCE=orm_test -e DB2INST1_PASSWORD=orm_test -e DBNAME=orm_test -e LICENSE=accept -e AUTOCONFIG=false -e ARCHIVE_LOGS=false -e TO_CREATE_SAMPLEDB=false -e REPODB=false -e BLU=false -e ENABLE_ORACLE_COMPATIBILITY=false -e UPDATEAVAIL=NO -e PERSISTENT_HOME=false -e HADR_ENABLED=false -p 50000:50000 $CONTAINER_OPTIONS -d ${DB_IMAGE_DB2_12_1:-icr.io/db2_community/db2:12.1.2.0} $CONTAINER_ARGS
     # Give the container some time to start
     OUTPUT=
     while [[ $OUTPUT != *"INSTANCE"* ]]; do
         echo "Waiting for DB2 to start..."
         sleep 10
-        OUTPUT=$($PRIVILEGED_CLI $CONTAINER_CLI logs db2 2>&1)
+        OUTPUT=$($CONTAINER_CLI logs db2 2>&1)
     done
-    $PRIVILEGED_CLI $CONTAINER_CLI exec -t db2 su - orm_test bash -c ". /database/config/orm_test/sqllib/db2profile; /database/config/orm_test/sqllib/bin/db2 'connect to orm_test'; /database/config/orm_test/sqllib/bin/db2 'CREATE USER TEMPORARY TABLESPACE usr_tbsp MANAGED BY AUTOMATIC STORAGE'"
+    $CONTAINER_CLI exec -t db2 su - orm_test bash -c ". /database/config/orm_test/sqllib/db2profile; /database/config/orm_test/sqllib/bin/db2 'connect to orm_test'; /database/config/orm_test/sqllib/bin/db2 'CREATE USER TEMPORARY TABLESPACE usr_tbsp MANAGED BY AUTOMATIC STORAGE'"
 }
 
 db2_spatial() {

documentation/src/main/asciidoc/userguide/chapters/query/extensions/Vector.adoc

@@ -53,15 +53,25 @@ so no further configuration is necessary to make the features available.
 [[vector-module-usage]]
 ==== Usage
 
-Annotate a persistent attribute with `@JdbcTypeCode(SqlTypes.VECTOR)` and specify the vector length with `@Array(length = ...)`.
+Annotate a persistent attribute with one of the various vector type codes `@JdbcTypeCode` and specify the vector length with `@Array(length = ...)`.
+Possible vector type codes and the compatible Java types are:
+
+* `@JdbcTypeCode(SqlTypes.VECTOR_BINARY)` for `byte[]`
+* `@JdbcTypeCode(SqlTypes.VECTOR_INT8)` for `byte[]`
+* `@JdbcTypeCode(SqlTypes.VECTOR_FLOAT16)` for `float[]`
+* `@JdbcTypeCode(SqlTypes.VECTOR_FLOAT32)` for `float[]`
+* `@JdbcTypeCode(SqlTypes.VECTOR_FLOAT64)` for `double[]`
+* `@JdbcTypeCode(SqlTypes.VECTOR)` for `float[]`
+
+Hibernate ORM also provides support for sparse vectors through dedicated Java types:
+
+* `@JdbcTypeCode(SqlTypes.SPARSE_VECTOR_INT8)` for `SparseByteVector`
+* `@JdbcTypeCode(SqlTypes.SPARSE_VECTOR_FLOAT32)` for `SparseFloatVector`
+* `@JdbcTypeCode(SqlTypes.SPARSE_VECTOR_FLOAT64)` for `SparseDoubleVector`
 
 [WARNING]
 ====
-As Oracle AI Vector Search supports different types of elements (to ensure better performance and compatibility with embedding models), you can also use:
-
-- `@JdbcTypeCode(SqlTypes.VECTOR_INT8)` for `byte[]`
-- `@JdbcTypeCode(SqlTypes.VECTOR_FLOAT32)` for `float[]`
-- `@JdbcTypeCode(SqlTypes.VECTOR_FLOAT64)` for `double[]`.
+Vector data type support depends on native support of the underlying database.
 ====
 
 [[vector-module-usage-example]]
@@ -88,14 +98,21 @@ Expressions of the vector type can be used with various vector functions.
 | `euclidean_distance()` | Computes the https://en.wikipedia.org/wiki/Euclidean_distance[euclidean distance] between two vectors. Maps to the `<``-``>` operator for `pgvector` and maps to the
 `vector_distance(v1, v2, EUCLIDEAN)` function for `Oracle AI Vector Search`.
 
+| `euclidean_squared_distance()` | Computes the https://en.wikipedia.org/wiki/Euclidean_distance#Squared_Euclidean_distance[squared euclidean distance] between two vectors.
+
 | `l2_distance()` | Alias for `euclidean_distance()`
 
+| `l2_squared_distance()` | Alias for `euclidean_squared_distance()`
+
 | `taxicab_distance()` | Computes the https://en.wikipedia.org/wiki/Taxicab_geometry[taxicab distance] between two vectors. Maps to `vector_distance(v1, v2, MANHATTAN)` function for `Oracle AI Vector Search`.
 
 | `l1_distance()` | Alias for `taxicab_distance()`
 
 | `hamming_distance()` | Computes the https://en.wikipedia.org/wiki/Hamming_distance[hamming distance] between two vectors. Maps to `vector_distance(v1, v2, HAMMING)` function for `Oracle AI Vector Search`.
 
+| `jaccard_distance()` | Computes the https://en.wikipedia.org/wiki/Jaccard_index[jaccard distance] between two vectors. Maps to the `<``%``>` operator for `pgvector` and maps to the
+`vector_distance(v1, v2, JACCARD)` function for `Oracle AI Vector Search`.
+
 | `inner_product()` | Computes the https://en.wikipedia.org/wiki/Inner_product_space[inner product] between two vectors
 
 | `negative_inner_product()` | Computes the negative inner product. Maps to the `<``#``>` operator for `pgvector` and maps to the
@@ -104,6 +121,14 @@ Expressions of the vector type can be used with various vector functions.
 | `vector_dims()` | Determines the dimensions of a vector
 
 | `vector_norm()` | Computes the https://en.wikipedia.org/wiki/Euclidean_space#Euclidean_norm[Euclidean norm] of a vector
+
+| `l2_norm()` | Alias for `vector_norm()`
+
+| `l2_normalize()` | Normalizes each component of a vector by dividing it with the https://en.wikipedia.org/wiki/Euclidean_space#Euclidean_norm[Euclidean norm] of the vector.
+
+| `binary_quantize()` | Reduces a vector of size N to a binary vector with N bits, using 0 for values <= 0 and 1 for values > 0.
+
+| `subvector()` | Creates a subvector from a given vector, a 1-based start index and a count.
 |===
 
 In addition to these special vector functions, it is also possible to use vectors with the following builtin `pgvector` operators:
@@ -143,6 +168,21 @@ include::{example-dir-vector}/FloatVectorTest.java[tags=euclidean-distance-examp
 ----
 ====
 
+[[vector-module-functions-euclidean-squared-distance]]
+===== `euclidean_squared_distance()` and `l2_squared_distance()`
+
+Computes the https://en.wikipedia.org/wiki/Euclidean_distance#Squared_Euclidean_distance[squared euclidean distance] between two vectors,
+which is `sum( (v1_i - v2_i)^2 )`, just like the regular `euclidean_distance`, but without the `sqrt`.
+The `l2_squared_distance()` function is an alias.
+
+[[vector-module-functions-euclidean-squared-distance-example]]
+====
+[source, java, indent=0]
+----
+include::{example-dir-vector}/FloatVectorTest.java[tags=euclidean-squared-distance-example]
+----
+====
+
 [[vector-module-functions-taxicab-distance]]
 ===== `taxicab_distance()` and `l1_distance()`
 
@@ -158,6 +198,36 @@ include::{example-dir-vector}/FloatVectorTest.java[tags=taxicab-distance-example
 ----
 ====
 
+[[vector-module-functions-hamming-distance]]
+===== `hamming_distance()`
+
+Computes the https://en.wikipedia.org/wiki/Hamming_distance[hamming distance] between two binary vectors,
+which is `bit_count(v1 ^ v2)` i.e. the amount of bits where two vectors differ.
+Maps to the `<``~``>` operator for `pgvector`.
+
+[[vector-module-functions-taxicab-distance-example]]
+====
+[source, java, indent=0]
+----
+include::{example-dir-vector}/BinaryVectorTest.java[tags=hamming-distance-example]
+----
+====
+
+[[vector-module-functions-jaccard-distance]]
+===== `jaccard_distance()`
+
+Computes the https://en.wikipedia.org/wiki/Jaccard_index[jaccard distance] between two binary vectors,
+which is `1 - bit_count(v1 & v2) / bit_count(v1 | v2)`.
+Maps to the `<``%``>` operator for `pgvector`.
+
+[[vector-module-functions-taxicab-distance-example]]
+====
+[source, java, indent=0]
+----
+include::{example-dir-vector}/BinaryVectorTest.java[tags=jaccard-distance-example]
+----
+====
+
 [[vector-module-functions-inner-product]]
 ===== `inner_product()` and `negative_inner_product()`
 
@@ -187,10 +257,11 @@ include::{example-dir-vector}/FloatVectorTest.java[tags=vector-dims-example]
 ====
 
 [[vector-module-functions-vector-norm]]
-===== `vector_norm()`
+===== `vector_norm()` and `l2_norm()`
 
 Computes the https://en.wikipedia.org/wiki/Euclidean_space#Euclidean_norm[Euclidean norm] of a vector,
 which is `sqrt( sum( v_i^2 ) )`.
+The `l2_norm()` function is an alias.
 
 [[vector-module-functions-vector-norm-example]]
 ====
@@ -200,6 +271,44 @@ include::{example-dir-vector}/FloatVectorTest.java[tags=vector-norm-example]
 ----
 ====
 
+[[vector-module-functions-l2-normalize]]
+===== `l2_normalize()`
+
+Normalizes each component of a vector by dividing it with the https://en.wikipedia.org/wiki/Euclidean_space#Euclidean_norm[Euclidean norm] of the vector.
+
+[[vector-module-functions-l2-normalize-example]]
+====
+[source, java, indent=0]
+----
+include::{example-dir-vector}/FloatVectorTest.java[tags=l2-normalize-example]
+----
+====
+
+[[vector-module-functions-binary-quantize]]
+===== `binary_quantize()`
+
+Reduces a vector of size N to a binary vector with N bits, using 0 for values <= 0 and 1 for values > 0.
+
+[[vector-module-functions-binary-quantize-example]]
+====
+[source, java, indent=0]
+----
+include::{example-dir-vector}/FloatVectorTest.java[tags=binary-quantize-example]
+----
+====
+
+[[vector-module-functions-subvector]]
+===== `binary_quantize()`
+
+Creates a subvector from a given vector, a 1-based start index and a count.
+
+[[vector-module-functions-subvector-example]]
+====
+[source, java, indent=0]
+----
+include::{example-dir-vector}/FloatVectorTest.java[tags=subvector-example]
+----
+====
 
 
 

hibernate-community-dialects/src/main/java/org/hibernate/community/dialect/GaussDBCastingInetJdbcType.java

@@ -10,7 +10,9 @@
 import java.sql.ResultSet;
 import java.sql.SQLException;
 
+import org.checkerframework.checker.nullness.qual.Nullable;
 import org.hibernate.dialect.Dialect;
+import org.hibernate.engine.jdbc.Size;
 import org.hibernate.sql.ast.spi.SqlAppender;
 import org.hibernate.type.SqlTypes;
 import org.hibernate.type.descriptor.ValueBinder;
@@ -35,6 +37,7 @@ public class GaussDBCastingInetJdbcType implements JdbcType {
 	@Override
 	public void appendWriteExpression(
 			String writeExpression,
+			@Nullable Size size,
 			SqlAppender appender,
 			Dialect dialect) {
 		appender.append( "cast(" );

hibernate-community-dialects/src/main/java/org/hibernate/community/dialect/GaussDBCastingIntervalSecondJdbcType.java

@@ -10,7 +10,9 @@
 import java.sql.ResultSet;
 import java.sql.SQLException;
 
+import org.checkerframework.checker.nullness.qual.Nullable;
 import org.hibernate.dialect.Dialect;
+import org.hibernate.engine.jdbc.Size;
 import org.hibernate.engine.spi.SessionFactoryImplementor;
 import org.hibernate.metamodel.mapping.JdbcMappingContainer;
 import org.hibernate.sql.ast.SqlAstTranslator;
@@ -77,6 +79,7 @@ public JdbcMappingContainer getExpressionType() {
 	@Override
 	public void appendWriteExpression(
 			String writeExpression,
+			@Nullable Size size,
 			SqlAppender appender,
 			Dialect dialect) {
 		appender.append( '(' );

hibernate-community-dialects/src/main/java/org/hibernate/community/dialect/GaussDBCastingJsonArrayJdbcType.java

@@ -4,7 +4,9 @@
  */
 package org.hibernate.community.dialect;
 
+import org.checkerframework.checker.nullness.qual.Nullable;
 import org.hibernate.dialect.Dialect;
+import org.hibernate.engine.jdbc.Size;
 import org.hibernate.sql.ast.spi.SqlAppender;
 import org.hibernate.type.descriptor.jdbc.JdbcType;
 import org.hibernate.type.descriptor.jdbc.JsonArrayJdbcType;
@@ -27,6 +29,7 @@ public GaussDBCastingJsonArrayJdbcType(JdbcType elementJdbcType, boolean jsonb)
 	@Override
 	public void appendWriteExpression(
 			String writeExpression,
+			@Nullable Size size,
 			SqlAppender appender,
 			Dialect dialect) {
 		appender.append( "cast(" );

hibernate-community-dialects/src/main/java/org/hibernate/community/dialect/GaussDBCastingJsonJdbcType.java

@@ -4,7 +4,9 @@
  */
 package org.hibernate.community.dialect;
 
+import org.checkerframework.checker.nullness.qual.Nullable;
 import org.hibernate.dialect.Dialect;
+import org.hibernate.engine.jdbc.Size;
 import org.hibernate.metamodel.mapping.EmbeddableMappingType;
 import org.hibernate.metamodel.spi.RuntimeModelCreationContext;
 import org.hibernate.sql.ast.spi.SqlAppender;
@@ -46,6 +48,7 @@ public AggregateJdbcType resolveAggregateJdbcType(
 	@Override
 	public void appendWriteExpression(
 			String writeExpression,
+			@Nullable Size size,
 			SqlAppender appender,
 			Dialect dialect) {
 		appender.append( "cast(" );