feat(flattening): allow flattening of n-ary structures in list (#1337)

a structured element which expands to `>=1` sub elements / foreign keys
can be expanded to all its leafs in a `list` or in a `group by`.

**Restrictions**:
- in other list like constructs like `function.args` & `order by` this
is only possible for structures with exactly one scalar leaf due to
unclear semantics.

The `getTransformedTokenStream` function has access to the context in
which the token stream shall be expanded. With this information it is
still possible reject a structure/assoc as a value in `having` & `where`

other changes:

feat(`flattening`): allow flattening of structures with exactly one leaf
in `order by` and `args`

Author: patricebender

db-service/lib/cqn4sql.js

@@ -96,7 +96,7 @@ function cqn4sql(originalQuery, model) {
 
     // Transform the existing where, prepend table aliases, and so on...
     if (where) {
-      transformedProp.where = getTransformedTokenStream(where)
+      transformedProp.where = getTransformedTokenStream(where, { prop: 'where' })
     }
 
     // Transform the from clause: association path steps turn into `WHERE EXISTS` subqueries.
@@ -191,7 +191,7 @@ function cqn4sql(originalQuery, model) {
 
     // Like the WHERE clause, aliases from the SELECT list are not accessible for `group by`/`having` (in most DB's)
     if (having) {
-      transformedQuery.SELECT.having = getTransformedTokenStream(having)
+      transformedQuery.SELECT.having = getTransformedTokenStream(having, { prop: 'having' })
     }
 
     if (groupBy) {
@@ -314,7 +314,7 @@ function cqn4sql(originalQuery, model) {
       lhs.args.push(arg)
       alreadySeen.set(nextAssoc.$refLink.alias, true)
       if (nextAssoc.where) {
-        const filter = getTransformedTokenStream(nextAssoc.where, nextAssoc.$refLink)
+        const filter = getTransformedTokenStream(nextAssoc.where, { $baseLink: nextAssoc.$refLink })
         lhs.on = [
           ...(hasLogicalOr(lhs.on) ? [asXpr(lhs.on)] : lhs.on),
           'and',
@@ -521,14 +521,14 @@ function cqn4sql(originalQuery, model) {
     }
   }
 
-  function resolveCalculatedElement(column, omitAlias = false, baseLink = null) {
+  function resolveCalculatedElement(column, omitAlias = false, $baseLink = null) {
     let value
 
     if (column.$refLinks) {
       const { $refLinks } = column
       value = $refLinks[$refLinks.length - 1].definition.value
       if (column.$refLinks.length > 1) {
-        baseLink =
+        $baseLink =
           [...$refLinks].reverse().find($refLink => $refLink.definition.isAssociation) ||
           // if there is no association in the path, the table alias is the base link
           // TA might refer to subquery -> we need to propagate the alias to all paths of the calc element
@@ -541,13 +541,13 @@ function cqn4sql(originalQuery, model) {
 
     let res
     if (ref) {
-      res = getTransformedTokenStream([value], baseLink)[0]
+      res = getTransformedTokenStream([value], { $baseLink })[0]
     } else if (xpr) {
-      res = { xpr: getTransformedTokenStream(value.xpr, baseLink) }
+      res = { xpr: getTransformedTokenStream(value.xpr, { $baseLink }) }
     } else if (val !== undefined) {
       res = { val }
     } else if (func) {
-      res = { args: getTransformedFunctionArgs(value.args, baseLink), func: value.func }
+      res = { args: getTransformedFunctionArgs(value.args, $baseLink), func: value.func }
     }
     if (!omitAlias) res.as = column.as || column.name || column.flatName
     return res
@@ -1018,7 +1018,7 @@ function cqn4sql(originalQuery, model) {
          * the result.
          */
         if (inOrderBy && flatColumns.length > 1)
-          throw new Error(`"${getFullName(leaf)}" can't be used in order by as it expands to multiple fields`)
+          throw new Error(`Structured element “${getFullName(leaf)}” expands to multiple fields and can't be used in order by`)
         flatColumns.forEach(fc => {
           if (col.nulls) fc.nulls = col.nulls
           if (col.sort) fc.sort = col.sort
@@ -1182,7 +1182,7 @@ function cqn4sql(originalQuery, model) {
     if (column.val || column.func || column.SELECT) return [column]
 
     const structsAreUnfoldedAlready = model.meta.unfolded?.includes('structs')
-    let { baseName, columnAlias = column.as, tableAlias } = names
+    let { baseName, columnAlias = column.as, tableAlias } = names || {}
     const { exclude, replace } = excludeAndReplace || {}
     const { $refLinks, flatName, isJoinRelevant } = column
     let firstNonJoinRelevantAssoc, stepAfterAssoc
@@ -1360,14 +1360,18 @@ function cqn4sql(originalQuery, model) {
    * @param {object[]} tokenStream - The token stream to transform. Each token in the stream is an
    *                                 object representing a CQN construct such as a column, an operator,
    *                                 or a subquery.
-   * @param {object} [$baseLink=null] - The context in which the `ref`s in the token stream are resolvable.
-   *                                    It serves as the reference point for resolving associations in
-   *                                    statements like `{…} WHERE exists assoc[exists anotherAssoc]`.
-   *                                    Here, the $baseLink for `anotherAssoc` would be `assoc`.
+   * @param {object} [context] - Optional context object.
+   * @param {object} [context.$baseLink] - The context in which the `ref`s in the token stream are resolvable.
+   *                                   It serves as the reference point for resolving associations in
+   *                                   statements like `{…} WHERE exists assoc[exists anotherAssoc]`.
+   *                                   Here, the $baseLink for `anotherAssoc` would be `assoc`.
+   * @param {string} [context.prop] - The query property which holds the token stream which shall be
+   *                                  transformed by this function, e.g. "where".
    * @returns {object[]} - The transformed token stream.
    */
-  function getTransformedTokenStream(tokenStream, $baseLink = null) {
+  function getTransformedTokenStream(tokenStream, context = {}) {
     const transformedTokenStream = []
+    const { $baseLink, /* prop */ } = context
     for (let i = 0; i < tokenStream.length; i++) {
       const token = tokenStream[i]
       if (token === 'exists') {
@@ -1453,7 +1457,7 @@ function cqn4sql(originalQuery, model) {
           if (list.every(e => e.val))
             // no need for transformation
             transformedTokenStream.push({ list })
-          else transformedTokenStream.push({ list: getTransformedTokenStream(list, $baseLink) })
+          else transformedTokenStream.push({ list: getTransformedTokenStream(list, { $baseLink, prop: 'list' }) })
         }
       } else if (tokenStream.length === 1 && token.val && $baseLink) {
         // infix filter - OData variant w/o mentioning key --> flatten out and compare each leaf to token.val
@@ -1509,13 +1513,13 @@ function cqn4sql(originalQuery, model) {
           i = indexRhs // jump to next relevant index
         } else {
           // reject associations in expression, except if we are in an infix filter -> $baseLink is set
-          assertNoStructInXpr(token, $baseLink)
+          assertNoStructInXpr(token, context)
           // reject virtual elements in expressions as they will lead to a sql error down the line
           if (lhsDef?.virtual) throw new Error(`Virtual elements are not allowed in expressions`)
 
           let result = is_regexp(token?.val) ? token : copy(token) // REVISIT: too expensive! //
           if (token.ref) {
-            const { definition } = token.$refLinks[token.$refLinks.length - 1]
+            const { definition } = token.$refLinks.at(-1)
             // Add definition to result
             setElementOnColumns(result, definition)
             if (isCalculatedOnRead(definition)) {
@@ -1536,7 +1540,15 @@ function cqn4sql(originalQuery, model) {
             const lastAssoc =
               token.isJoinRelevant && [...token.$refLinks].reverse().find(l => l.definition.isAssociation)
             const tableAlias = getTableAlias(token, (!lastAssoc?.onlyForeignKeyAccess && lastAssoc) || $baseLink)
-            if ((!$baseLink || lastAssoc) && token.isJoinRelevant) {
+            if(isAssocOrStruct(definition)) {
+              const flat =  getFlatColumnsFor(token, { tableAlias: $baseLink?.alias || tableAlias })
+              if(flat.length === 0)
+                throw new Error(`Structured element “${getFullName(definition)}” expands to nothing and can't be used in expressions`)
+              else if (flat.length > 1 && context.prop !== 'list') // only acceptable in `list`
+                throw new Error(`Structured element “${getFullName(definition)}” expands to multiple fields and can't be used in expressions`)
+              transformedTokenStream.push(...flat)
+              continue
+            } else if ((!$baseLink || lastAssoc) && token.isJoinRelevant) {
               let name = calculateElementName(token)
               result.ref = [tableAlias, name]
             } else if (tableAlias) {
@@ -1549,7 +1561,7 @@ function cqn4sql(originalQuery, model) {
             result = transformSubquery(token)
           } else {
             if (token.xpr) {
-              result.xpr = getTransformedTokenStream(token.xpr, $baseLink)
+              result.xpr = getTransformedTokenStream(token.xpr, { $baseLink })
             }
             if (token.func && token.args) {
               result.args = getTransformedFunctionArgs(token.args, $baseLink)
@@ -1661,11 +1673,15 @@ function cqn4sql(originalQuery, model) {
     }
   }
 
-  function assertNoStructInXpr(token, inInfixFilter = false) {
-    if (!inInfixFilter && token.$refLinks?.[token.$refLinks.length - 1].definition.target)
-      // REVISIT: let this through if not requested otherwise
+  function assertNoStructInXpr(token, context) {
+    const definition = token.$refLinks?.at(-1).definition
+    if(!definition) return
+    const rejectStructs = context && (context.prop in { where: 1, having: 1 })
+    // unmanaged is always forbidden
+    // expanding a ref in a `where`/`having` context
+    if ((rejectStructs && definition?.target) || definition?.on)
       rejectAssocInExpression()
-    if (isStructured(token.$refLinks?.[token.$refLinks.length - 1].definition))
+    if (rejectStructs && isStructured(definition))
       // REVISIT: let this through if not requested otherwise
       rejectStructInExpression()
 
@@ -1771,7 +1787,7 @@ function cqn4sql(originalQuery, model) {
 
       // only append infix filter to outer where if it is the leaf of the from ref
       if (refReverse[0].where)
-        filterConditions.push(getTransformedTokenStream(refReverse[0].where, $refLinksReverse[0]))
+        filterConditions.push(getTransformedTokenStream(refReverse[0].where,{ $baseLink: $refLinksReverse[0] }))
 
       if (existingWhere.length > 0) filterConditions.push(existingWhere)
       if (whereExistsSubSelects.length > 0) {
@@ -2209,7 +2225,7 @@ function cqn4sql(originalQuery, model) {
     }
 
     if (customWhere) {
-      const filter = getTransformedTokenStream(customWhere, next)
+      const filter = getTransformedTokenStream(customWhere, { $baseLink: next })
       const wrappedFilter = hasLogicalOr(filter) ? [asXpr(filter)] : filter
       on.push('and', ...wrappedFilter)
     }
@@ -2315,10 +2331,10 @@ function cqn4sql(originalQuery, model) {
   function getTransformedFunctionArgs(args, $baseLink = null) {
     let result = null
     if (Array.isArray(args)) {
-      result = args.map(t => {
+      result = args.flatMap(t => {
         if (!t.val)
           // this must not be touched
-          return getTransformedTokenStream([t], $baseLink)[0]
+          return getTransformedTokenStream([t], { $baseLink })
         return t
       })
     } else if (typeof args === 'object') {
@@ -2327,7 +2343,7 @@ function cqn4sql(originalQuery, model) {
         const t = args[prop]
         if (!t.val)
           // this must not be touched
-          result[prop] = getTransformedTokenStream([t], $baseLink)[0]
+          result[prop] = getTransformedTokenStream([t], { $baseLink })[0]
         else result[prop] = t
       }
     }

db-service/test/bookshop/db/schema.cds

@@ -106,6 +106,9 @@ entity Bar {
       x : Integer;
     };
   };
+  empty: {
+    // nothing
+  }
 }
 
 entity EStruc {

db-service/test/bookshop/db/search.cds

@@ -123,3 +123,17 @@ entity BookShelf {
         books : Composition of many Books
                     on books.shelf = $self;
 }
+
+@cds.search: {
+    toMulti
+}
+entity MultipleLeafAssocAsKey {
+    key toMulti : Association to MultipleKeys;
+}
+
+entity MultipleKeys {
+    key ID1 : Integer;
+    key ID2 : Integer;
+    key ID3 : Integer;
+    text: String;
+}