HHH-18661 Add unnest() set-returning function and enable XML/JSON based array support on more databases

Author: beikov

docker_db.sh

@@ -434,6 +434,8 @@ use master
 go
 create login $SYBASE_USER with password $SYBASE_PASSWORD
 go
+exec sp_configure 'enable xml', 1
+go
 exec sp_dboption $SYBASE_DB, 'abort tran on log full', true
 go
 exec sp_dboption $SYBASE_DB, 'allow nulls by default', true

documentation/src/main/asciidoc/userguide/chapters/query/hql/QueryLanguage.adoc

@@ -8,6 +8,7 @@
 :array-example-dir-hql: {core-project-dir}/src/test/java/org/hibernate/orm/test/function/array
 :json-example-dir-hql: {core-project-dir}/src/test/java/org/hibernate/orm/test/function/json
 :xml-example-dir-hql: {core-project-dir}/src/test/java/org/hibernate/orm/test/function/xml
+:srf-example-dir-hql: {core-project-dir}/src/test/java/org/hibernate/orm/test/function/srf
 :extrasdir: extras
 
 This chapter describes Hibernate Query Language (HQL) and Jakarta Persistence Query Language (JPQL).
@@ -1197,32 +1198,33 @@ The following functions deal with SQL array types, which are not supported on ev
 |===
 | Function | Purpose
 
-| `array()` | Creates an array based on the passed arguments
-| `array_list()` | Like `array`, but returns the result as `List<?>`
-| `array_agg()` | Aggregates row values into an array
-| `array_position()` | Determines the position of an element in an array
-| `array_positions()` | Determines all positions of an element in an array
-| `array_positions_list()` | Like `array_positions`, but returns the result as `List<Integer>`
-| `array_length()` | Determines the length of an array
-| `array_concat()` | Concatenates array with each other in order
-| `array_prepend()` | Prepends element to array
-| `array_append()` | Appends element to array
-| `array_contains()` | Whether an array contains an element
-| `array_contains_nullable()` | Whether an array contains an element, supporting `null` element
-| `array_includes()` | Whether an array contains another array
-| `array_includes_nullable()` | Whether an array contains another array, supporting `null` elements
-| `array_intersects()` | Whether an array holds at least one element of another array
-| `array_intersects_nullable()` | Whether an array holds at least one element of another array, supporting `null` elements
-| `array_get()` | Accesses the element of an array by index
-| `array_set()` | Creates array copy with given element at given index
-| `array_remove()` | Creates array copy with given element removed
-| `array_remove_index()` | Creates array copy with the element at the given index removed
-| `array_slice()` | Creates a sub-array of the based on lower and upper index
-| `array_replace()` | Creates array copy replacing a given element with another
-| `array_trim()` | Creates array copy trimming the last _N_ elements
-| `array_fill()` | Creates array filled with the same element _N_ times
-| `array_fill_list()` | Like `array_fill`, but returns the result as `List<?>`
-| `array_to_string()` | String representation of array
+| <<hql-array-constructor-functions,`array()`>> | Creates an array based on the passed arguments
+| <<hql-array-constructor-functions,`array_list()`>> | Like `array`, but returns the result as `List<?>`
+| <<hql-array-aggregate-functions,`array_agg()`>> | Aggregates row values into an array
+| <<hql-array-position-functions,`array_position()`>> | Determines the position of an element in an array
+| <<hql-array-positions-functions,`array_positions()`>> | Determines all positions of an element in an array
+| <<hql-array-positions-functions,`array_positions_list()`>> | Like `array_positions`, but returns the result as `List<Integer>`
+| <<hql-array-length-functions,`array_length()`>> | Determines the length of an array
+| <<hql-array-concat-functions,`array_concat()`>> | Concatenates array with each other in order
+| <<hql-array-prepend-functions,`array_prepend()`>> | Prepends element to array
+| <<hql-array-append-functions,`array_append()`>> | Appends element to array
+| <<hql-array-contains-functions,`array_contains()`>> | Whether an array contains an element
+| <<hql-array-contains-functions,`array_contains_nullable()`>> | Whether an array contains an element, supporting `null` element
+| <<hql-array-includes-example,`array_includes()`>> | Whether an array contains another array
+| <<hql-array-includes-example,`array_includes_nullable()`>> | Whether an array contains another array, supporting `null` elements
+| <<hql-array-intersects-functions,`array_intersects()`>> | Whether an array holds at least one element of another array
+| <<hql-array-intersects-functions,`array_intersects_nullable()`>> | Whether an array holds at least one element of another array, supporting `null` elements
+| <<hql-array-get-functions,`array_get()`>> | Accesses the element of an array by index
+| <<hql-array-set-functions,`array_set()`>> | Creates array copy with given element at given index
+| <<hql-array-remove-functions,`array_remove()`>> | Creates array copy with given element removed
+| <<hql-array-remove-index-functions,`array_remove_index()`>> | Creates array copy with the element at the given index removed
+| <<hql-array-slice-functions,`array_slice()`>> | Creates a sub-array of the based on lower and upper index
+| <<hql-array-replace-functions,`array_replace()`>> | Creates array copy replacing a given element with another
+| <<hql-array-trim-functions,`array_trim()`>> | Creates array copy trimming the last _N_ elements
+| <<hql-array-fill-functions,`array_fill()`>> | Creates array filled with the same element _N_ times
+| <<hql-array-fill-functions,`array_fill_list()`>> | Like `array_fill`, but returns the result as `List<?>`
+| <<hql-array-to-string-functions,`array_to_string()`>> | String representation of array
+| <<hql-array-unnest,`unnest()`>> | Turns an array into rows
 |===
 
 [[hql-array-constructor-functions]]
@@ -1637,6 +1639,32 @@ include::{array-example-dir-hql}/ArrayToStringTest.java[tags=hql-array-to-string
 ----
 ====
 
+[[hql-array-unnest]]
+===== `unnest()`
+
+A <<hql-from-set-returning-functions,set-returning function>>, which turns the single array argument into rows.
+Returns no rows if the array argument is `null` or an empty array.
+The `index()` function can be used to access the 1-based array index of an array element.
+
+[[hql-array-unnest-struct-example]]
+====
+[source, java, indent=0]
+----
+include::{array-example-dir-hql}/ArrayUnnestStructTest.java[tags=hql-array-unnest-aggregate-with-ordinality-example]
+----
+====
+
+The `lateral` keyword is mandatory if the argument refers to a from node item of the same query level.
+Basic plural attributes can also be joined directly, which is syntax sugar for `lateral unnest(...)`.
+
+[[hql-array-unnest-example]]
+====
+[source, java, indent=0]
+----
+include::{array-example-dir-hql}/ArrayUnnestTest.java[tags=hql-array-unnest-example]
+----
+====
+
 [[hql-functions-json]]
 ==== Functions for dealing with JSON
 
@@ -2916,6 +2944,48 @@ The CTE name can be used for a `from` clause root or a `join`, similar to entity
 
 Refer to the <<hql-with-cte,with clause>> chapter for details about CTEs.
 
+[[hql-from-set-returning-functions]]
+==== Set-returning functions in `from` clause
+
+A set-returning function is a function that produces rows instead of a single scalar value
+and is exclusively used in the `from` clause, either as root node or join target.
+
+The `index()` function can be used to access the 1-based index of a returned row.
+
+The following set-returning functions are available on many platforms:
+
+|===
+| Function | purpose
+
+| <<hql-array-unnest,`unnest()`>> | Turns an array into rows
+//| `generate_series()` | Creates a series of values as rows
+|===
+
+To use set returning functions defined in the database, it is required to register them in a `FunctionContributor`:
+
+[[hql-from-set-returning-functions-contributor-example]]
+====
+[source, java, indent=0]
+----
+include::{srf-example-dir-hql}/CustomSetReturningFunctionTest.java[tags=hql-set-returning-function-contributor-example]
+----
+====
+
+After that, the function can be used in the `from` clause:
+
+[[hql-from-set-returning-functions-custom-example]]
+====
+[source, java, indent=0]
+----
+include::{srf-example-dir-hql}/CustomSetReturningFunctionTest.java[tags=hql-set-returning-function-custom-example]
+----
+====
+
+NOTE: The `index()` function represents the idea of the `with ordinality` SQL syntax,
+which is not supported on some databases for user defined functions.
+Hibernate ORM tries to emulate this feature by wrapping invocations as lateral subqueries and using `row_number()`,
+which may lead to worse performance.
+
 [[hql-join]]
 === Declaring joined entities
 
@@ -3131,6 +3201,17 @@ Most databases support some flavor of `join lateral`, and Hibernate emulates the
 But emulation is neither very efficient, nor does it support all possible query shapes, so it's important to test on your target database.
 ====
 
+[[hql-join-set-returning-function]]
+==== Set-returning functions in joins
+
+A `join` clause may contain a set-returning function, either:
+
+- an uncorrelated set-returning function, which is almost the same as a <<hql-from-set-returning-functions,set-returning function in the `from` clause>>, except that it may have an `on` restriction, or
+- a _lateral join_, which is a correlated set-returning function, and may refer to other roots declared earlier in the same `from` clause.
+
+The `lateral` keyword just distinguishes the two cases.
+A lateral join may be an inner or left outer join, but not a right join, nor a full join.
+
 [[hql-implicit-join]]
 ==== Implicit association joins (path expressions)
 

hibernate-community-dialects/src/main/java/org/hibernate/community/dialect/CockroachLegacyDialect.java

@@ -18,6 +18,7 @@
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+import org.checkerframework.checker.nullness.qual.Nullable;
 import org.hibernate.LockMode;
 import org.hibernate.LockOptions;
 import org.hibernate.PessimisticLockException;
@@ -92,7 +93,6 @@
 import static org.hibernate.type.SqlTypes.INET;
 import static org.hibernate.type.SqlTypes.INTEGER;
 import static org.hibernate.type.SqlTypes.JSON;
-import static org.hibernate.type.SqlTypes.JSON_ARRAY;
 import static org.hibernate.type.SqlTypes.LONG32NVARCHAR;
 import static org.hibernate.type.SqlTypes.LONG32VARBINARY;
 import static org.hibernate.type.SqlTypes.LONG32VARCHAR;
@@ -263,11 +263,9 @@ protected void registerColumnTypes(TypeContributions typeContributions, ServiceR
 		if ( getVersion().isSameOrAfter( 20 ) ) {
 			ddlTypeRegistry.addDescriptor( new DdlTypeImpl( INET, "inet", this ) );
 			ddlTypeRegistry.addDescriptor( new DdlTypeImpl( JSON, "jsonb", this ) );
-			ddlTypeRegistry.addDescriptor( new DdlTypeImpl( JSON_ARRAY, "jsonb", this ) );
 		}
 		else {
 			ddlTypeRegistry.addDescriptor( new DdlTypeImpl( JSON, "json", this ) );
-			ddlTypeRegistry.addDescriptor( new DdlTypeImpl( JSON_ARRAY, "json", this ) );
 		}
 		ddlTypeRegistry.addDescriptor( new NamedNativeEnumDdlTypeImpl( this ) );
 		ddlTypeRegistry.addDescriptor( new NamedNativeOrdinalEnumDdlTypeImpl( this ) );
@@ -372,23 +370,23 @@ protected void contributeCockroachTypes(TypeContributions typeContributions, Ser
 				if ( getVersion().isSameOrAfter( 20, 0 ) ) {
 					jdbcTypeRegistry.addDescriptorIfAbsent( PgJdbcHelper.getInetJdbcType( serviceRegistry ) );
 					jdbcTypeRegistry.addDescriptorIfAbsent( PgJdbcHelper.getJsonbJdbcType( serviceRegistry ) );
-					jdbcTypeRegistry.addDescriptorIfAbsent( PgJdbcHelper.getJsonbArrayJdbcType( serviceRegistry ) );
+					jdbcTypeRegistry.addTypeConstructorIfAbsent( PgJdbcHelper.getJsonbArrayJdbcType( serviceRegistry ) );
 				}
 				else {
 					jdbcTypeRegistry.addDescriptorIfAbsent( PgJdbcHelper.getJsonJdbcType( serviceRegistry ) );
-					jdbcTypeRegistry.addDescriptorIfAbsent( PgJdbcHelper.getJsonArrayJdbcType( serviceRegistry ) );
+					jdbcTypeRegistry.addTypeConstructorIfAbsent( PgJdbcHelper.getJsonArrayJdbcType( serviceRegistry ) );
 				}
 			}
 			else {
 				jdbcTypeRegistry.addDescriptorIfAbsent( PostgreSQLCastingIntervalSecondJdbcType.INSTANCE );
 				if ( getVersion().isSameOrAfter( 20, 0 ) ) {
 					jdbcTypeRegistry.addDescriptorIfAbsent( PostgreSQLCastingInetJdbcType.INSTANCE );
 					jdbcTypeRegistry.addDescriptorIfAbsent( PostgreSQLCastingJsonJdbcType.JSONB_INSTANCE );
-					jdbcTypeRegistry.addDescriptorIfAbsent( PostgreSQLCastingJsonArrayJdbcType.JSONB_INSTANCE );
+					jdbcTypeRegistry.addTypeConstructorIfAbsent( PostgreSQLCastingJsonArrayJdbcTypeConstructor.JSONB_INSTANCE );
 				}
 				else {
 					jdbcTypeRegistry.addDescriptorIfAbsent( PostgreSQLCastingJsonJdbcType.JSON_INSTANCE );
-					jdbcTypeRegistry.addDescriptorIfAbsent( PostgreSQLCastingJsonArrayJdbcType.JSON_INSTANCE );
+					jdbcTypeRegistry.addTypeConstructorIfAbsent( PostgreSQLCastingJsonArrayJdbcTypeConstructor.JSON_INSTANCE );
 				}
 			}
 		}
@@ -398,11 +396,11 @@ protected void contributeCockroachTypes(TypeContributions typeContributions, Ser
 			if ( getVersion().isSameOrAfter( 20, 0 ) ) {
 				jdbcTypeRegistry.addDescriptorIfAbsent( PostgreSQLCastingInetJdbcType.INSTANCE );
 				jdbcTypeRegistry.addDescriptorIfAbsent( PostgreSQLCastingJsonJdbcType.JSONB_INSTANCE );
-				jdbcTypeRegistry.addDescriptorIfAbsent( PostgreSQLCastingJsonArrayJdbcType.JSONB_INSTANCE );
+				jdbcTypeRegistry.addTypeConstructorIfAbsent( PostgreSQLCastingJsonArrayJdbcTypeConstructor.JSONB_INSTANCE );
 			}
 			else {
 				jdbcTypeRegistry.addDescriptorIfAbsent( PostgreSQLCastingJsonJdbcType.JSON_INSTANCE );
-				jdbcTypeRegistry.addDescriptorIfAbsent( PostgreSQLCastingJsonArrayJdbcType.JSON_INSTANCE );
+				jdbcTypeRegistry.addTypeConstructorIfAbsent( PostgreSQLCastingJsonArrayJdbcTypeConstructor.JSON_INSTANCE );
 			}
 		}
 
@@ -424,6 +422,7 @@ protected void contributeCockroachTypes(TypeContributions typeContributions, Ser
 				)
 		);
 
+		// Replace the standard array constructor
 		jdbcTypeRegistry.addTypeConstructor( PostgreSQLArrayJdbcTypeConstructor.INSTANCE );
 	}
 
@@ -518,6 +517,8 @@ public void initializeFunctionRegistry(FunctionContributions functionContributio
 		functionFactory.jsonArrayAppend_postgresql( false );
 		functionFactory.jsonArrayInsert_postgresql();
 
+		functionFactory.unnest_postgresql();
+
 		// Postgres uses # instead of ^ for XOR
 		functionContributions.getFunctionRegistry().patternDescriptorBuilder( "bitxor", "(?1#?2)" )
 				.setExactArgumentCount( 2 )
@@ -534,6 +535,11 @@ public void initializeFunctionRegistry(FunctionContributions functionContributio
 		functionContributions.getFunctionRegistry().registerAlternateKey( "truncate", "trunc" );
 	}
 
+	@Override
+	public @Nullable String getDefaultOrdinalityColumnName() {
+		return "ordinality";
+	}
+
 	@Override
 	public TimeZoneSupport getTimeZoneSupport() {
 		return TimeZoneSupport.NORMALIZE;

hibernate-community-dialects/src/main/java/org/hibernate/community/dialect/DB2LegacyDialect.java

@@ -88,6 +88,7 @@
 import org.hibernate.tool.schema.extract.internal.SequenceInformationExtractorNoOpImpl;
 import org.hibernate.tool.schema.extract.spi.SequenceInformationExtractor;
 import org.hibernate.type.JavaObjectType;
+import org.hibernate.type.SqlTypes;
 import org.hibernate.type.StandardBasicTypes;
 import org.hibernate.type.descriptor.ValueExtractor;
 import org.hibernate.type.descriptor.java.JavaType;
@@ -455,6 +456,14 @@ public void initializeFunctionRegistry(FunctionContributions functionContributio
 			functionFactory.xmlexists_db2_legacy();
 		}
 		functionFactory.xmlagg();
+
+		functionFactory.unnest_emulated();
+	}
+
+	@Override
+	public int getPreferredSqlTypeCodeForArray() {
+		// Even if DB2 11 supports JSON functions, it's not possible to unnest a JSON array to rows, so stick to XML
+		return SqlTypes.XML_ARRAY;
 	}
 
 	@Override

hibernate-community-dialects/src/main/java/org/hibernate/community/dialect/DB2LegacySqlAstTranslator.java

@@ -10,15 +10,19 @@
 import org.hibernate.LockMode;
 import org.hibernate.dialect.DatabaseVersion;
 import org.hibernate.engine.spi.SessionFactoryImplementor;
+import org.hibernate.metamodel.mapping.CollectionPart;
 import org.hibernate.metamodel.mapping.JdbcMappingContainer;
+import org.hibernate.metamodel.mapping.ModelPart;
 import org.hibernate.query.IllegalQueryOperationException;
+import org.hibernate.query.derived.AnonymousTupleTableGroupProducer;
 import org.hibernate.query.sqm.ComparisonOperator;
 import org.hibernate.query.sqm.FetchClauseType;
 import org.hibernate.sql.ast.Clause;
 import org.hibernate.sql.ast.SqlAstNodeRenderingMode;
 import org.hibernate.sql.ast.spi.AbstractSqlAstTranslator;
 import org.hibernate.sql.ast.spi.SqlSelection;
 import org.hibernate.sql.ast.tree.MutationStatement;
+import org.hibernate.sql.ast.tree.SqlAstNode;
 import org.hibernate.sql.ast.tree.Statement;
 import org.hibernate.sql.ast.tree.delete.DeleteStatement;
 import org.hibernate.sql.ast.tree.expression.CaseSearchedExpression;
@@ -28,6 +32,8 @@
 import org.hibernate.sql.ast.tree.expression.Literal;
 import org.hibernate.sql.ast.tree.expression.SqlTuple;
 import org.hibernate.sql.ast.tree.expression.Summarization;
+import org.hibernate.sql.ast.tree.from.DerivedTableReference;
+import org.hibernate.sql.ast.tree.from.FunctionTableReference;
 import org.hibernate.sql.ast.tree.from.NamedTableReference;
 import org.hibernate.sql.ast.tree.from.QueryPartTableReference;
 import org.hibernate.sql.ast.tree.from.TableGroup;
@@ -253,6 +259,34 @@ public void visitQueryPartTableReference(QueryPartTableReference tableReference)
 		inLateral = oldLateral;
 	}
 
+	@Override
+	protected void renderDerivedTableReference(DerivedTableReference tableReference) {
+		if ( tableReference instanceof FunctionTableReference && tableReference.isLateral() ) {
+			// No need for a lateral keyword for functions
+			tableReference.accept( this );
+		}
+		else {
+			super.renderDerivedTableReference( tableReference );
+		}
+	}
+
+	@Override
+	public void renderNamedSetReturningFunction(String functionName, List<? extends SqlAstNode> sqlAstArguments, AnonymousTupleTableGroupProducer tupleType, String tableIdentifierVariable, SqlAstNodeRenderingMode argumentRenderingMode) {
+		final ModelPart ordinalitySubPart = tupleType.findSubPart( CollectionPart.Nature.INDEX.getName(), null );
+		if ( ordinalitySubPart != null ) {
+			appendSql( "lateral (select t.*, row_number() over() " );
+			appendSql( ordinalitySubPart.asBasicValuedModelPart().getSelectionExpression() );
+			appendSql( " from table(" );
+			renderSimpleNamedFunction( functionName, sqlAstArguments, argumentRenderingMode );
+			append( ") t)" );
+		}
+		else {
+			appendSql( "table(" );
+			super.renderNamedSetReturningFunction( functionName, sqlAstArguments, tupleType, tableIdentifierVariable, argumentRenderingMode );
+			append( ')' );
+		}
+	}
+
 	@Override
 	public void visitSelectStatement(SelectStatement statement) {
 		if ( getQueryPartForRowNumbering() == statement.getQueryPart() && inLateral ) {