feat: guarantee a well-defined order for all flexSearch queries

A flexSearch query without orderBy will now use the primarySort (flexSearchOrder) as order. If no flexSearchOrder is configured, this defaults to createdAt_DESC, id_DESC. If flexSearchOrder is configured, createdAt_DESC, id_DESC will always be appended to make order deterministic.

A flexSearch query with an explicit orderBy that contradicts with the primarySort (so primarySort cannot be used) will also use an absolute order now.

The performance effect should be relatively minor because using primary sort is very efficient.

Author: Yogu

src/model/implementation/root-entity-type.ts

@@ -179,6 +179,8 @@ export class RootEntityType extends ObjectTypeBase {
     @memorize()
     get discriminatorField(): Field {
         // later, we can return @key here when it exists and is required
+        // however, consider: this is used everywhere in primarySort, so changing it would cause
+        // migrations to re-create all flexsearch views
         return this.getFieldOrThrow(ID_FIELD);
     }
 

src/schema-generation/flex-search-generator.ts

@@ -222,8 +222,10 @@ export class FlexSearchGenerator {
             ...schemaField,
             transform: (sourceNode, args, context) => {
                 const assertionVariable = new VariableQueryNode();
-                // If a filter or an order_by is specified, a pre-execution query node is added that throws a TOO_MANY_OBJECTS_ERROR if the amount of objects the filter or order_by is
-                // used on is to large
+                // If a filter or an order_by is specified, a pre-execution query node is added
+                // that throws a TOO_MANY_OBJECTS_ERROR if the amount of objects returned by the
+                // flexSearchFilter is too large. This prevents performance issues with
+                // in-memory filtering and sorting.
                 const filterArg = args[POST_FILTER_ARG] || args[FILTER_ARG];
                 if (
                     (filterArg && Object.keys(filterArg).length > 0) ||

src/schema-generation/order-by-and-pagination-augmentation.ts

@@ -45,6 +45,10 @@ import { QueryNodeField } from './query-node-object-type';
 import { RootFieldHelper } from './root-field-helper';
 import { and, binaryOp, binaryOpWithAnaylzer } from './utils/input-types';
 import { getOrderByValues } from './utils/pagination';
+import {
+    getSortClausesForPrimarySort,
+    orderArgMatchesPrimarySort,
+} from './utils/flex-search-utils';
 
 export enum LimitTypeCheckType {
     RESULT_VALIDATOR = 'RESULT_VALIDATOR',
@@ -201,17 +205,40 @@ export class OrderByAndPaginationAugmentation {
                     // ok. If the sorting matches the primary sort, sorting is efficient, but you still need to specify
                     // it, otherwise, sometimes the order can be wrong (after inserting or updating documents).
 
-                    // this may generate a SORT clause that is not covered by the flexsearch index,
-                    // but the TooManyObject check of flex-search-generator already handles this case to throw
-                    // a TooManyObjects error if needed.
-                    orderByValues = getOrderByValues(args, orderByType, {
-                        isAbsoluteOrderRequired,
-                    });
+                    // flexsearch is mostly used in tables, where a consistent absolute sort order is important
+                    // At the same time, primary sort is relatively cheap to use (in a test with primed caches on
+                    // 10 million objects, slowdown was 9% (after warmup), and the whole operation was still sub-millisecond)
+                    // for this reason, we guarantee an absolute, well-defined sort ordering at all times in flexsearch
+                    // - if possible (does not contradict orderBy), we use the primarySort, which is very fast
+                    // - if not possible, sorting will happen in-memory anyway, so it does not hurt to make it absolute
+                    if (
+                        orderArgMatchesPrimarySort(
+                            args[ORDER_BY_ARG],
+                            listNode.rootEntityType.flexSearchPrimarySort,
+                        )
+                    ) {
+                        // note that the primary sort always has an absolute order
+                        // (there is special logic in RootEntityType to guarantee this)
+                        orderByValues = getSortClausesForPrimarySort(
+                            listNode.rootEntityType,
+                            orderByType,
+                        );
+                    } else {
+                        // this will generate a SORT clause that is not covered by the primary sort,
+                        // so sorting will happen purely in memory.
+                        // FlexSearchGenerator.augmentWithCondition will add a check to throw a
+                        // FLEX_SEARCH_TOO_MANY_OBJECTS error if there are too many objects to sort in memory
+                        // (it has the same orderArgMatchesPrimarySort-based logic)
+                        orderByValues = getOrderByValues(args, orderByType, {
+                            // Since we're already sorting in-memory, it does not hurt to make this ordering absolute
+                            isAbsoluteOrderRequired: true,
+                        });
+                    }
 
-                    // for now, cursor-based pagination is only allowed when we can use flexsearch filters for the
-                    // paginationFilter. This simplifies the implementation, but maybe we should support it in the
-                    // future for consistency. Then however we need to adjust the too-many-objects check
-                    // do this check also if cursor is just requested so we get this error on the first page
+                    // Cursor-based pagination is only allowed when we can use flexsearch filters for the
+                    // paginationFilter. Otherwise, we would need a postFilter for the "after" arg,
+                    // and that could lead to bad performance / TOO_MANY_OBJECTS error. Better be
+                    // consistent and outright disallow cursor-based pagination in this case
                     if (isCursorRequested || afterArg) {
                         const violatingClauses = orderByValues.filter((val) =>
                             val.path.some((field) => !field.isFlexSearchIndexed),

src/schema-generation/output-type-generator.ts

@@ -36,7 +36,10 @@ import {
     QueryNodeOutputType,
 } from './query-node-object-type';
 import { RootFieldHelper } from './root-field-helper';
-import { orderArgMatchesPrimarySort } from './utils/flex-search-utils';
+import {
+    getSortClausesForPrimarySort,
+    orderArgMatchesPrimarySort,
+} from './utils/flex-search-utils';
 import { getOrderByValues } from './utils/pagination';
 
 export class OutputTypeGenerator {
@@ -159,13 +162,7 @@ export class OutputTypeGenerator {
                 objectType.flexSearchPrimarySort,
             )
         ) {
-            // this would be cleaner if the primary sort was actually parsed into a ModelComponent (see e.g. the Index and IndexField classes)
-            orderByValues = objectType.flexSearchPrimarySort.map((clause) =>
-                orderByType.getValueOrThrow(
-                    clause.field.path.replace('.', '_') +
-                        (clause.direction === OrderDirection.ASCENDING ? '_ASC' : '_DESC'),
-                ),
-            );
+            orderByValues = getSortClausesForPrimarySort(objectType, orderByType);
         } else {
             orderByValues = getOrderByValues(listFieldRequest.args, orderByType, {
                 isAbsoluteOrderRequired: true,

src/schema-generation/utils/flex-search-utils.ts

@@ -1,11 +1,13 @@
 import { FlexSearchPrimarySortClause } from '../../model/implementation/flex-search';
 import { OrderDirection } from '../../model/implementation/order';
+import { RootEntityType } from '../../model';
+import { OrderByEnumType, OrderByEnumValue } from '../order-by-enum-generator';
 
 export function orderArgMatchesPrimarySort(
     clauses: ReadonlyArray<string> | undefined,
     primarySort: ReadonlyArray<FlexSearchPrimarySortClause>,
 ): boolean {
-    // TODO what about sort clauses that are added automatically because the user used cursor-based pagination?
+    // if no orderBy is given, we will always default to the primarySort order (OrderByAndPaginationAugmentation)
     if (!clauses || !clauses.length) {
         return true;
     }
@@ -14,6 +16,8 @@ export function orderArgMatchesPrimarySort(
         return false;
     }
 
+    // note that primary sort cannot be used backwards
+
     for (let index = 0; index < clauses.length; index++) {
         const arg = clauses[index];
         if (arg.endsWith('_ASC')) {
@@ -37,3 +41,19 @@ export function orderArgMatchesPrimarySort(
 
     return true;
 }
+
+/**
+ * Generates a list of orderBy enum values that exactly represent the primary search
+ */
+export function getSortClausesForPrimarySort(
+    objectType: RootEntityType,
+    orderByType: OrderByEnumType,
+): ReadonlyArray<OrderByEnumValue> {
+    // this would be cleaner if the primary sort was actually parsed into a ModelComponent (see e.g. the Index and IndexField classes)
+    return objectType.flexSearchPrimarySort.map((clause) =>
+        orderByType.getValueOrThrow(
+            clause.field.path.replace('.', '_') +
+                (clause.direction === OrderDirection.ASCENDING ? '_ASC' : '_DESC'),
+        ),
+    );
+}