From 921db29d18423f4412df984419af5fbb5759dff9 Mon Sep 17 00:00:00 2001 From: Robin De Schepper Date: Mon, 24 Mar 2025 17:46:32 +0100 Subject: [PATCH 1/9] added helper that can find nested primary columns --- src/helper.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/helper.ts b/src/helper.ts index 61278e5f1..1d872a380 100644 --- a/src/helper.ts +++ b/src/helper.ts @@ -206,6 +206,19 @@ export function checkIsNestedRelation(qb: SelectQueryBuilder, propertyP return true } +export function checkIsOneOfNestedPrimaryColumns(qb: SelectQueryBuilder, propertyPath: string): boolean { + let metadata = qb?.expressionMap?.mainAlias?.metadata + const [deepestProperty, ...subRelations] = propertyPath.split('.').reverse() + for (const relationName of subRelations.reverse()) { + const relation = metadata?.relations.find((relation) => relation.propertyPath === relationName) + if (!relation) { + return false + } + metadata = relation.inverseEntityMetadata + } + return !!metadata.primaryColumns.find(col => col.propertyName === deepestProperty) +} + export function checkIsEmbedded(qb: SelectQueryBuilder, propertyPath: string): boolean { if (!qb || !propertyPath) { return false From 37344afcf45b97ed9d1123211529b3e4ff659871 Mon Sep 17 00:00:00 2001 From: Robin De Schepper Date: Mon, 24 Mar 2025 17:51:14 +0100 Subject: [PATCH 2/9] reinstated support for absence checking via null filter on PK --- src/filter.ts | 40 +++++++++++++++++++++++++++------------- 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/src/filter.ts b/src/filter.ts index 628df998b..5260e89db 100644 --- a/src/filter.ts +++ b/src/filter.ts @@ -23,6 +23,7 @@ import { checkIsEmbedded, checkIsJsonb, checkIsNestedRelation, + checkIsOneOfNestedPrimaryColumns, checkIsRelation, extractVirtualProperty, fixColumnAlias, @@ -392,17 +393,30 @@ export function addFilter( ) } - // Set the join type of every relationship used in a filter to `innerJoinAndSelect` - // so that records without that relationships don't show up in filters on their columns. - return Object.fromEntries( - filterEntries - .map(([key]) => [key, getPropertiesByColumnName(key)] as const) - .filter(([, properties]) => properties.propertyPath) - .flatMap(([, properties]) => { - const nesting = properties.column.split('.') - return Array.from({ length: nesting.length - 1 }, (_, i) => nesting.slice(0, i + 1).join('.')) - .filter((relation) => checkIsNestedRelation(qb, relation)) - .map((relation) => [relation, 'innerJoinAndSelect'] as const) - }) - ) + const columnJoinMethods: ColumnJoinMethods = {} + const nullFilteredRelations = [] + for (const [key, columnFilters] of filterEntries) { + const properties = getPropertiesByColumnName(key) + const relationPath = properties.column.split('.') + for (let i = 0; i < relationPath.length - 1; i++) { + const subRelation = relationPath.slice(0, i + 1).join('.') + if (!checkIsNestedRelation(qb, subRelation)) continue + columnJoinMethods[subRelation] = 'innerJoinAndSelect' + } + + // When a $null filter is set on a primary key of a relationship, + // the filter acts as an absence filter (i.e., filters on results + // that do not have the child relationship). We mark these columns + // to be left joined later so that `rel.pk IS NULL` works. + if ( + checkIsOneOfNestedPrimaryColumns(qb, properties.column) && + columnFilters.some((filter) => filter.findOperator.type === 'isNull') + ) { + nullFilteredRelations.push(relationPath.slice(0, -1).join('.')) + } + } + for (const column of nullFilteredRelations) { + columnJoinMethods[column] = 'leftJoinAndSelect' + } + return columnJoinMethods } From 70ea087b170a4dece44b374c64800c8eeb4687f5 Mon Sep 17 00:00:00 2001 From: Robin De Schepper Date: Mon, 24 Mar 2025 17:54:20 +0100 Subject: [PATCH 3/9] added test for $null PK absence checking --- src/paginate.spec.ts | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/paginate.spec.ts b/src/paginate.spec.ts index b1ac53d86..c92943cdc 100644 --- a/src/paginate.spec.ts +++ b/src/paginate.spec.ts @@ -3357,6 +3357,30 @@ describe('paginate', () => { }) }) + describe('$null operator', () => { + it('should find cats without any toys when applied on PK', async () => { + const config: PaginateConfig = { + relations: ['toys'], + sortableColumns: ['id', 'toys.id'], + filterableColumns: { + 'toys.id': [FilterOperator.NULL] + } + } + const query: PaginateQuery = { + filter: { + // Null-filtering a relationship's PK should check + // for absence of the relationship + 'toys.id': '$null' + }, + path: '', + } + + const result = await paginate(query, catRepo, config); + // Should find only cats without toys. + expect(result.data.every(cat => cat.toys.length === 0)).toBe(true); + }) + }) + describe('should correctly handle number column filter', () => { it('with $eq operator and valid number', async () => { const config: PaginateConfig = { From ea92d183a1f8dd796809ade8dd4fdd956d4cad3e Mon Sep 17 00:00:00 2001 From: Robin De Schepper Date: Fri, 24 Oct 2025 21:23:03 +0200 Subject: [PATCH 4/9] switched to-many relationships to an exists-based system --- src/filter.ts | 270 ++++++++++++++++++++++++++++++++++++++----- src/paginate.spec.ts | 202 +++++++++++++++++++++++++++++++- src/paginate.ts | 8 +- 3 files changed, 441 insertions(+), 39 deletions(-) diff --git a/src/filter.ts b/src/filter.ts index 5260e89db..3b50b75eb 100644 --- a/src/filter.ts +++ b/src/filter.ts @@ -3,6 +3,7 @@ import { ArrayContains, Between, Brackets, + EntityMetadata, Equal, FindOperator, ILike, @@ -22,16 +23,19 @@ import { checkIsArray, checkIsEmbedded, checkIsJsonb, - checkIsNestedRelation, - checkIsOneOfNestedPrimaryColumns, checkIsRelation, + createRelationSchema, extractVirtualProperty, fixColumnAlias, getPropertiesByColumnName, isDateColumnType, isISODate, JoinMethod, + mergeRelationSchema, } from './helper' +import { EmbeddedMetadata } from 'typeorm/metadata/EmbeddedMetadata' +import { RelationMetadata } from 'typeorm/metadata/RelationMetadata' +import { addRelationsFromSchema } from './paginate' export enum FilterOperator { EQ = '$eq', @@ -52,6 +56,7 @@ export function isOperator(value: unknown): value is FilterOperator { } export enum FilterSuffix { + // Used to negate a filter NOT = '$not', } @@ -192,10 +197,12 @@ export function addWhereCondition(qb: SelectQueryBuilder, column: string, ) { condition.parameters[0] = `cardinality(${condition.parameters[0]})` } + const expression = qb['createWhereConditionExpression'](condition) if (columnFilter.comparator === FilterComparator.OR) { - qb.orWhere(qb['createWhereConditionExpression'](condition), parameters) + qb.orWhere(expression, parameters) } else { - qb.andWhere(qb['createWhereConditionExpression'](condition), parameters) + console.log('expression', expression, parameters) + qb.andWhere(expression, parameters) } }) } @@ -366,16 +373,100 @@ export function parseFilter( return filter } +class RelationPathError extends Error {} + +/** + * Retrieves the relation path for a given column name within the provided metadata. + * + * This method analyzes the column name's segments to identify corresponding relations or embedded entities + * within the hierarchy described by the given metadata and returns a structured path. + * + * @param {string} columnName - The dot-delimited name of the column whose relation path is to be determined. + * @param {EntityMetadata | EmbeddedMetadata} metadata - The metadata of the entity or embedded component + * which holds the relations or embedded items. + * @return {[string, RelationMetadata | EmbeddedMetadata][]} The ordered array describing the path, + * where each element contains a field name and its corresponding relation or embedded metadata. + * Throws an error if no matching relation or embedded metadata is found. + */ +export function getRelationPath( + columnName: string, + metadata: EntityMetadata | EmbeddedMetadata +): [string, RelationMetadata | EmbeddedMetadata][] { + const relationSegments = columnName.split('.') + const deeper = relationSegments.slice(1).join('.') + const fieldName = relationSegments[0].replace(/[()]/g, '') + + try { + // Check if there's a relation with this property name + const relation = metadata.relations.find((r) => r.propertyName === fieldName) + if (relation) { + return [ + [fieldName, relation] as const, + ...(relationSegments.length > 1 ? getRelationPath(deeper, relation.inverseEntityMetadata) : []), + ] + } + + // Check if there's something embedded with this property name + const embedded = metadata.embeddeds.find((embedded) => embedded.propertyName === fieldName) + if (embedded) { + return [ + [fieldName, embedded] as const, + ...(relationSegments.length > 1 ? getRelationPath(deeper, embedded) : []), + ] + } + } catch (e) { + if (e instanceof RelationPathError) { + throw new RelationPathError(`No relation or embedded found for property path ${columnName}`) + } + throw e + } + if (relationSegments.length > 1) + throw new RelationPathError(`No relation or embedded found for property path ${columnName}`) + return [] +} + +/** + * Finds the first 'to-many' relationship in a given entity metadata or embedded metadata + * given a column name. A 'to-many' relationship can be either a one-to-many or a + * many-to-many relationship. + * + * @param {string} columnName - The column name to traverse through its segments and find relationships. + * @param {EntityMetadata | EmbeddedMetadata} metadata - The metadata of the entity or + * embedded object in which relationships are defined. + * @return {{ path: string[]; relation: RelationMetadata } | undefined} An object containing + * the path to the 'to-many' relationship and the relationship metadata, or undefined if no + * 'to-many' relationships are found. + */ +function findFirstToManyRelationship( + columnName: string, + metadata: EntityMetadata | EmbeddedMetadata +): { path: string[]; relation: RelationMetadata } | undefined { + const relationPath = getRelationPath(columnName, metadata) + const relationSegments = columnName.split('.') + const firstToMany = relationPath.findIndex( + ([, relation]) => 'isOneToMany' in relation && (relation.isOneToMany || relation.isManyToMany) + ) + if (firstToMany > -1) + return { + path: relationSegments.slice(0, firstToMany + 1), + relation: relationPath[firstToMany][1] as RelationMetadata, + } +} + export function addFilter( qb: SelectQueryBuilder, query: PaginateQuery, - filterableColumns?: { [column: string]: (FilterOperator | FilterSuffix)[] | true } -): ColumnJoinMethods { + filterableColumns?: { [column: string]: (FilterOperator | FilterSuffix | FilterQuantifier)[] | true } +) { + const mainMetadata = qb.expressionMap.mainAlias.metadata const filter = parseFilter(query, filterableColumns, qb) const filterEntries = Object.entries(filter) - const orFilters = filterEntries.filter(([_, value]) => value[0].comparator === '$or') - const andFilters = filterEntries.filter(([_, value]) => value[0].comparator === '$and') + + // Looks for filters that don't have a toMany relationship on their path: will be translated to WHERE clauses on this query. + const whereFilters = filterEntries.filter(([key]) => !findFirstToManyRelationship(key, mainMetadata)) + const orFilters = whereFilters.filter(([, value]) => value[0].comparator === '$or') + const andFilters = whereFilters.filter(([, value]) => value[0].comparator === '$and') qb.andWhere( new Brackets((qb: SelectQueryBuilder) => { @@ -393,30 +484,153 @@ export function addFilter( ) } + // Looks filters that have a toMany relationship on their path: will be translated to EXISTS subqueries. + const existsFilters = filterEntries.map(([key]) => findFirstToManyRelationship(key, mainMetadata)).filter(Boolean) + // Find all the different toMany starting paths + const toManyPaths = [...new Set(existsFilters.map((f) => f.path.join('.')))] + const toManyRelations = toManyPaths.map( + (path) => [path, existsFilters.find((f) => f.path.join('.') === path).relation] as const + ) + + for (const [path, relation] of toManyRelations) { + const mainQueryAlias = qb.alias + const relationPath = getRelationPath(path, mainMetadata) + const toManyAlias = `_rel_${relationPath[relationPath.length - 1][0]}_${relationPath.length - 1}` + + // 1. Create the EXISTS subquery, starting from the toMany entity + const existsQb = qb.connection.createQueryBuilder(relation.inverseEntityMetadata.target as any, toManyAlias) + + // 2. Add the subfilters to the EXISTS subquery + const { subQuery, subFilterableColumns } = createSubFilter(query, filterableColumns, path) + const subJoins = addFilter(existsQb, subQuery, subFilterableColumns) + + // 3. Add the sub relationship joins to the EXISTS subquery + const relationsSchema = mergeRelationSchema(createRelationSchema(Object.keys(subJoins))) + // Only inner join, no need to select anything + addRelationsFromSchema(existsQb, relationsSchema, {}, 'innerJoin') + + // 4. Build the chain of joins that backtracks our toMany relationship to the root. + + // We iterate from the second-to-last item (the immediate parent) back to the first item (root's child) + for (let i = relationPath.length - 1; i >= 0; i--) { + const [, meta] = relationPath[i] + + // --- A: Skip Embedded Entities --- + if (meta.type === 'embedded') { + // Embedded entities exist within the current table (currentChildAlias). + // They do not require a JOIN, so we simply continue to the next item in the path. + continue + } + + // --- B: Handle Table Relation (RelationMetadata) --- + // If we reach this point, 'meta' is a RelationMetadata object. + const parentMeta = meta as RelationMetadata + + // Check if this is an intermediate or top-level relationship + if (i !== 0) { + // --- Intermediate Join (Joining two non-root tables within the subquery) --- + + const parentAlias = `_rel_${relationPath[i - 1][0]}_${i - 1}` + const childAlias = `_rel_${relationPath[i][0]}_${i}` + const childRelationMetadata = parentMeta.inverseRelation + + // Get the join columns from the inverse relation of the current metadata (e.g., inverse of 'pillows') + const joinCols = childRelationMetadata.joinColumns + + // Construct the ON condition (handles composite keys) + const onConditions = joinCols + .map((jc) => { + const fk = jc.databaseName + const pk = jc.referencedColumn.databaseName + return `"${childAlias}"."${fk}" = "${parentAlias}"."${pk}"` + }) + .join(' AND ') + + // Add the INNER JOIN to the subquery + existsQb.innerJoin( + childRelationMetadata.inverseRelation.entityMetadata.target, + parentAlias, + onConditions + ) + } else { + // Perform the final correlation WHERE clause to the main query alias + const joinMeta = parentMeta.isOwning ? parentMeta : parentMeta.inverseRelation + for (const joinColumn of joinMeta.joinColumns) { + // 1. Get the raw column names from the owning metadata: + const fkColumn = joinColumn.databaseName + const pkColumn = joinColumn.referencedColumn.databaseName + + // 2. Get the table aliases + let fkAlias: string + let pkAlias: string + if (parentMeta.isOwning) { + pkAlias = `_rel_${relationPath[0][0]}_0` + fkAlias = mainQueryAlias + } else { + fkAlias = `_rel_${relationPath[0][0]}_0` + pkAlias = mainQueryAlias + } + + // Correlation + existsQb.andWhere(`"${fkAlias}"."${fkColumn}" = "${pkAlias}"."${pkColumn}"`) + } + } + } + + // 5. Add the EXISTS subquery to the main query + qb.andWhereExists(existsQb) + } + const columnJoinMethods: ColumnJoinMethods = {} - const nullFilteredRelations = [] - for (const [key, columnFilters] of filterEntries) { - const properties = getPropertiesByColumnName(key) - const relationPath = properties.column.split('.') - for (let i = 0; i < relationPath.length - 1; i++) { - const subRelation = relationPath.slice(0, i + 1).join('.') - if (!checkIsNestedRelation(qb, subRelation)) continue - columnJoinMethods[subRelation] = 'innerJoinAndSelect' + for (const [key] of filterEntries) { + const relationPath = getRelationPath(key, mainMetadata) + for (let i = 0; i < relationPath.length; i++) { + const [, subRelation] = relationPath[i] + const column = relationPath + .slice(0, i + 1) + .map((p) => p[0]) + .join('.') + // Join the toMany + if ('isOneToMany' in subRelation) { + if (subRelation.isOneToOne || subRelation.isManyToOne) { + columnJoinMethods[column] = 'innerJoinAndSelect' + } else { + // Stop traversing at toMany boundaries, since those will be handled by EXISTS subqueries + break + } + } } + } - // When a $null filter is set on a primary key of a relationship, - // the filter acts as an absence filter (i.e., filters on results - // that do not have the child relationship). We mark these columns - // to be left joined later so that `rel.pk IS NULL` works. - if ( - checkIsOneOfNestedPrimaryColumns(qb, properties.column) && - columnFilters.some((filter) => filter.findOperator.type === 'isNull') - ) { - nullFilteredRelations.push(relationPath.slice(0, -1).join('.')) + return columnJoinMethods +} + +export function createSubFilter( + query: PaginateQuery, + filterableColumns: { [column: string]: (FilterOperator | FilterSuffix | FilterQuantifier)[] | true }, + column: string +) { + const subQuery = { filter: {} } as PaginateQuery + for (const [subColumn, filter] of Object.entries(query.filter)) { + if (subColumn.startsWith(column + '.')) { + subQuery.filter[getSubColumn(column, subColumn)] = filter } } - for (const column of nullFilteredRelations) { - columnJoinMethods[column] = 'leftJoinAndSelect' + const subFilterableColumns: { [column: string]: (FilterOperator | FilterSuffix | FilterQuantifier)[] | true } = {} + for (const [subColumn, filter] of Object.entries(filterableColumns)) { + if (subColumn.startsWith(column + '.')) { + subFilterableColumns[getSubColumn(column, subColumn)] = filter + } } - return columnJoinMethods + return { subQuery, subFilterableColumns } +} + +function getSubColumn(column: string, subColumn: string) { + const sliced = subColumn.slice(column.length + 1) + if (sliced.startsWith('(') && sliced.endsWith(')')) { + // Embedded relationships need to be unpacked from subColumn.(embedded.property) to + // embedded.property + return sliced.slice(1, -1) + } + return sliced } diff --git a/src/paginate.spec.ts b/src/paginate.spec.ts index c92943cdc..96d9252cf 100644 --- a/src/paginate.spec.ts +++ b/src/paginate.spec.ts @@ -1,7 +1,7 @@ import { HttpException, Logger } from '@nestjs/common' import { clone } from 'lodash' import * as process from 'process' -import { DataSource, In, Like, Repository, TypeORMError } from 'typeorm' +import { DataSource, In, Like, Repository, SelectQueryBuilder, TypeORMError } from 'typeorm' import { BaseDataSourceOptions } from 'typeorm/data-source/BaseDataSourceOptions' import { CatHairEntity } from './__tests__/cat-hair.entity' import { CatHomePillowBrandEntity } from './__tests__/cat-home-pillow-brand.entity' @@ -3363,21 +3363,23 @@ describe('paginate', () => { relations: ['toys'], sortableColumns: ['id', 'toys.id'], filterableColumns: { - 'toys.id': [FilterOperator.NULL] - } + 'toys.id': [FilterOperator.NULL], + }, } const query: PaginateQuery = { filter: { // Null-filtering a relationship's PK should check // for absence of the relationship - 'toys.id': '$null' + 'toys.id': '$null', }, path: '', } - const result = await paginate(query, catRepo, config); + const result = await paginate(query, catRepo, config) // Should find only cats without toys. - expect(result.data.every(cat => cat.toys.length === 0)).toBe(true); + expect(result.data.every((cat) => cat.toys.length === 0)).toBe(true) + // Should find 5 cats without toys. + expect(result.data.length).toBe(5) }) }) @@ -4973,4 +4975,192 @@ describe('paginate', () => { expect(result.data[0].toys[0]).not.toHaveProperty('createdAt') }) }) + + describe('Filtering across to-many relationship boundaries', () => { + let existsSpy + + beforeAll(() => { + existsSpy = jest.spyOn(SelectQueryBuilder.prototype, 'andWhereExists') + }) + + beforeEach(() => { + existsSpy.mockClear() + }) + + afterAll(() => { + existsSpy.mockRestore() + }) + + describe('Filtering records whose related entities match filter criteria', () => { + it('should find all cats that have one or more toys that are not toy 0', async () => { + // This test tests a direct toMany relationship (.toys) with a single direct filter on it (.toys.id) + const config: PaginateConfig = { + relations: ['toys'], + sortableColumns: ['id', 'toys.id'], + filterableColumns: { + 'toys.id': [FilterOperator.EQ, FilterSuffix.NOT], + 'toys.(size.height)': [FilterOperator.GT], + 'home.name': [FilterOperator.EQ], + }, + } + const query: PaginateQuery = { + filter: { + // Filtering on toMany means "include cats with toys that match the filter", + // in this case "include cats with toys that are not toys with the id of toy 0" + 'toys.id': `$not:$eq:${catToys[0].id}`, + }, + path: '', + } + + const result = await paginate(query, catRepo, config) + // Cat 0 has toys 0, 1, 2 --> is included because it has toy 1 and 2 which are not toy 0 + // Cat 1 has toys 3 --> is included + // Other cats have no toys --> are not included + expect(result.data.length).toBe(2) + expect(result.data[0].id).toBe(cats[0].id) + expect(result.data[1].id).toBe(cats[1].id) + + // When filtering on toMany relations, the related entities themselves should not be filtered. + expect(result.data[0].toys.length).toBe(catToys.filter((t) => t.cat.id === cats[0].id).length) + }) + + it('should find all cats with one or more toys height 5 that is also not toy 0', async () => { + // This test tests a direct toMany relationship (.toys) with multiple filters on it. + // It tests that all filters are applied so that only toys match that meet all filters (rather than + // all the toys that meet one or more filter criteria), and + // it asserts that only a single optimized EXISTS clause is generated. + const config: PaginateConfig = { + relations: ['toys'], + sortableColumns: ['id', 'toys.id'], + filterableColumns: { + 'toys.id': [FilterOperator.EQ, FilterSuffix.NOT], + 'toys.(size.height)': [FilterOperator.EQ], + }, + } + const query: PaginateQuery = { + filter: { + 'toys.id': `$not:$eq:${catToys[0].id}`, + 'toys.(size.height)': '$eq:5', + }, + path: '', + } + + const result = await paginate(query, catRepo, config) + // Both cat 0 and cat 1 have a toy that is not toy 0, but only cat 0 has a toy with height == 5 + expect(result.data.length).toBe(1) + expect(result.data[0].id).toBe(cats[0].id) + + // When filtering on toMany relations, the related entities themselves should not be filtered. + expect(result.data[0].toys.length).toBe(catToys.filter((t) => t.cat.id === cats[0].id).length) + + // Only a single EXISTS clause should be generated + expect(existsSpy).toHaveBeenCalledTimes(1) + }) + + it('should find cats with toys, even when that relationship is not loaded', async () => { + // This test tests a regression where filtering by a relationship that did not occur in the `relations` + // config would cause the query to fail. e.g. filter on `toys.id` without `toys` in `relations` + const config: PaginateConfig = { + sortableColumns: ['id'], + filterableColumns: { + 'toys.(size.height)': [FilterOperator.EQ], + }, + } + const query: PaginateQuery = { + filter: { + 'toys.(size.height)': '$eq:5', + }, + path: '', + } + + const result = await paginate(query, catRepo, config) + // Only cat 0 has a toy with height == 5 + expect(result.data.length).toBe(1) + expect(result.data[0].id).toBe(cats[0].id) + // Filtering by a relationship should not include it in the result + expect(result.data[0].toys).toBeUndefined() + }) + + it('should find all cats with one or more red or teal pillows in their home', async () => { + // This test tests toMany relationships that are part of a deeper chain such as cat.home.pillows + const config: PaginateConfig = { + sortableColumns: ['id'], + relations: ['home.pillows'], + filterableColumns: { + 'home.pillows.color': [FilterOperator.EQ], + }, + } + const query: PaginateQuery = { + filter: { + 'home.pillows.color': [`$or:$eq:red`, `$or:$eq:teal`], + }, + path: '', + } + + const result = await paginate(query, catRepo, config) + // Cat 0 has a red pillow in their home, and cat 1 has a teal pillow in their home + expect(result.data.length).toBe(2) + expect(result.data[0].id).toBe(cats[0].id) + expect(result.data[1].id).toBe(cats[1].id) + + // When filtering on toMany relations, the related entities themselves should not be filtered. + expect(result.data[0].home.pillows.length).toBe(3) + + // Only a single EXISTS clause should be generated + expect(existsSpy).toHaveBeenCalledTimes(1) + }) + + it('should find all cats with a toy from the shop on main street', async () => { + // This test tests that the exists clauses can still correctly deal with nested relations + // e.g. toys.shop.address.address + // ^ the toMany relationship with a tail of nested relations + const config: PaginateConfig = { + sortableColumns: ['id'], + filterableColumns: { + 'toys.shop.address.address': [FilterOperator.ILIKE], + }, + } + const query: PaginateQuery = { + filter: { + 'toys.shop.address.address': [`$ilike:main`], + }, + path: '', + } + + const result = await paginate(query, catRepo, config) + // Cat 1 has a toy from the shop on main street + expect(result.data.length).toBe(1) + expect(result.data[0].id).toBe(cats[0].id) + + // Only a single EXISTS clause should be generated + expect(existsSpy).toHaveBeenCalledTimes(1) + }) + + it('should find all cats with a tall toy and a red pillow in their home', async () => { + // This test tests filtering on multiple toMany relationships and asserts the number of EXISTS clauses + const config: PaginateConfig = { + sortableColumns: ['id'], + filterableColumns: { + 'home.pillows.color': [FilterOperator.EQ], + 'toys.(size.height)': [FilterOperator.GT], + }, + } + const query: PaginateQuery = { + filter: { + 'home.pillows.color': [`$eq:red`], + 'toys.(size.height)': '$gt:5', + }, + path: '', + } + + const result = await paginate(query, catRepo, config) + // Cat 0 has a red pillow in their home, and cat 1 has a teal pillow in their home + expect(result.data.length).toBe(1) + expect(result.data[0].id).toBe(cats[0].id) + + // 2 EXISTS clauses should be generated, one for each toMany relationship used in the filters + expect(existsSpy).toHaveBeenCalledTimes(2) + }) + }) + }) }) diff --git a/src/paginate.ts b/src/paginate.ts index 204b6ac0d..98c467f95 100644 --- a/src/paginate.ts +++ b/src/paginate.ts @@ -671,7 +671,7 @@ export async function paginate( createRelationSchema(config.relations), createRelationSchema(Object.keys(joinMethods)) ) - addRelationsFromSchema(queryBuilder, relationsSchema, config, joinMethods) + addRelationsFromSchema(queryBuilder, relationsSchema, joinMethods, config.defaultJoinMethod) } if (config.paginationType !== PaginationType.CURSOR) { @@ -1023,11 +1023,9 @@ export async function paginate( export function addRelationsFromSchema( queryBuilder: SelectQueryBuilder, schema: RelationSchema, - config: PaginateConfig, - joinMethods: Partial> + joinMethods: Partial>, + defaultJoinMethod: 'leftJoin' | 'innerJoin' | 'leftJoinAndSelect' | 'innerJoinAndSelect' = 'leftJoinAndSelect', ): void { - const defaultJoinMethod = config.defaultJoinMethod ?? 'leftJoinAndSelect' - const createQueryBuilderRelations = ( prefix: string, relations: RelationSchema, From 1db4e253d10bc5e34c02bd001e8307fba24fc608 Mon Sep 17 00:00:00 2001 From: Robin De Schepper Date: Sat, 25 Oct 2025 17:08:41 +0200 Subject: [PATCH 5/9] added a delete column to confirm tomany filters respect soft delete --- src/__tests__/cat-home-pillow.entity.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/__tests__/cat-home-pillow.entity.ts b/src/__tests__/cat-home-pillow.entity.ts index f64beb403..fea72de19 100644 --- a/src/__tests__/cat-home-pillow.entity.ts +++ b/src/__tests__/cat-home-pillow.entity.ts @@ -1,4 +1,4 @@ -import { Column, CreateDateColumn, Entity, ManyToOne, PrimaryGeneratedColumn } from 'typeorm' +import { Column, CreateDateColumn, DeleteDateColumn, Entity, ManyToOne, PrimaryGeneratedColumn } from 'typeorm' import { CatHomePillowBrandEntity } from './cat-home-pillow-brand.entity' import { CatHomeEntity } from './cat-home.entity' import { DateColumnNotNull } from './column-option' @@ -19,4 +19,7 @@ export class CatHomePillowEntity { @CreateDateColumn(DateColumnNotNull) createdAt: string + + @DeleteDateColumn(DateColumnNotNull) + deletedAt: string } From 308dfa65818f0d09d583813d1665ce74584c3945 Mon Sep 17 00:00:00 2001 From: Robin De Schepper Date: Sat, 25 Oct 2025 17:10:14 +0200 Subject: [PATCH 6/9] added quantifiers to toMany filtering --- src/filter.ts | 139 ++++++++++++++++++++++++++++++++----------- src/helper.ts | 38 +++++++++++- src/paginate.spec.ts | 132 +++++++++++++++++++++++++++++++++++++++- src/paginate.ts | 6 +- 4 files changed, 274 insertions(+), 41 deletions(-) diff --git a/src/filter.ts b/src/filter.ts index 3b50b75eb..2a117af6b 100644 --- a/src/filter.ts +++ b/src/filter.ts @@ -20,6 +20,8 @@ import { import { WherePredicateOperator } from 'typeorm/query-builder/WhereClause' import { PaginateQuery } from './decorator' import { + andWhereAllExist, + andWhereNoneExist, checkIsArray, checkIsEmbedded, checkIsJsonb, @@ -64,6 +66,16 @@ export function isSuffix(value: unknown): value is FilterSuffix { return values(FilterSuffix).includes(value as any) } +export enum FilterQuantifier { + ALL = '$all', + ANY = '$any', + NONE = '$none', +} + +export function isQuantifier(value: unknown): value is FilterQuantifier { + return values(FilterQuantifier).includes(value as any) +} + export enum FilterComparator { AND = '$and', OR = '$or', @@ -91,11 +103,12 @@ export const OperatorSymbolToFunction = new Map< [FilterOperator.CONTAINS, ArrayContains], ]) -type Filter = { comparator: FilterComparator; findOperator: FindOperator } +type Filter = { quantifier: FilterQuantifier; comparator: FilterComparator; findOperator: FindOperator } type ColumnFilters = { [columnName: string]: Filter[] } type ColumnJoinMethods = { [columnName: string]: JoinMethod } export interface FilterToken { + quantifier: FilterQuantifier comparator: FilterComparator suffix?: FilterSuffix operator: FilterOperator @@ -201,7 +214,6 @@ export function addWhereCondition(qb: SelectQueryBuilder, column: string, if (columnFilter.comparator === FilterComparator.OR) { qb.orWhere(expression, parameters) } else { - console.log('expression', expression, parameters) qb.andWhere(expression, parameters) } }) @@ -213,22 +225,25 @@ export function parseFilterToken(raw?: string): FilterToken | null { } const token: FilterToken = { + quantifier: FilterQuantifier.ANY, comparator: FilterComparator.AND, suffix: undefined, operator: FilterOperator.EQ, value: raw, } - const MAX_OPERTATOR = 4 // max 4 operator es: $and:$not:$eq:$null + const MAX_OPERATOR = 5 // max 5 operator: $none:$and:$not:$eq:$null const OPERAND_SEPARATOR = ':' const matches = raw.split(OPERAND_SEPARATOR) - const maxOperandCount = matches.length > MAX_OPERTATOR ? MAX_OPERTATOR : matches.length - const notValue: (FilterOperator | FilterSuffix | FilterComparator)[] = [] + const maxOperandCount = matches.length > MAX_OPERATOR ? MAX_OPERATOR : matches.length + const notValue: (FilterOperator | FilterSuffix | FilterComparator | FilterQuantifier)[] = [] for (let i = 0; i < maxOperandCount; i++) { const match = matches[i] - if (isComparator(match)) { + if (isQuantifier(match)) { + token.quantifier = match + } else if (isComparator(match)) { token.comparator = match } else if (isSuffix(match)) { token.suffix = match @@ -241,10 +256,11 @@ export function parseFilterToken(raw?: string): FilterToken | null { } if (notValue.length) { - token.value = - token.operator === FilterOperator.NULL - ? undefined - : raw.replace(`${notValue.join(OPERAND_SEPARATOR)}${OPERAND_SEPARATOR}`, '') + token.value = !raw.includes(OPERAND_SEPARATOR) + ? // things like `$null`, `$none`, and `$any`, have no token value + undefined + : // otherwise, remove the operators and separators from the raw string to obtain the token value + raw.replace(`${notValue.join(OPERAND_SEPARATOR)}${OPERAND_SEPARATOR}`, '') } return token @@ -270,7 +286,7 @@ function fixColumnFilterValue(column: string, qb: SelectQueryBuilder, isJs export function parseFilter( query: PaginateQuery, - filterableColumns?: { [column: string]: (FilterOperator | FilterSuffix)[] | true }, + filterableColumns?: { [column: string]: (FilterOperator | FilterSuffix | FilterQuantifier)[] | true }, qb?: SelectQueryBuilder ): ColumnFilters { const filter: ColumnFilters = {} @@ -307,9 +323,13 @@ export function parseFilter( if (token.suffix && !allowedOperators.includes(token.suffix)) { continue } + if (token.quantifier !== FilterQuantifier.ANY && !allowedOperators.includes(token.quantifier)) { + continue + } } const params: (typeof filter)[0][0] = { + quantifier: token.quantifier, comparator: token.comparator, findOperator: undefined, } @@ -347,6 +367,7 @@ export function parseFilter( const jsonFixValue = fixColumnFilterValue(column, qb, true) const jsonParams = { + quantifier: params.quantifier, comparator: params.comparator, findOperator: JsonContains({ [jsonColumnName]: jsonFixValue(token.value), @@ -461,10 +482,44 @@ export function addFilter( const mainMetadata = qb.expressionMap.mainAlias.metadata const filter = parseFilter(query, filterableColumns, qb) + addDirectFilters(qb, filter) + addToManySubFilters(qb, filter, query, filterableColumns) + + // Direct filters require to be joined, so pass the join information back up to the main pagination builder + // (or the parent filter query in case of subfilters) + const columnJoinMethods: ColumnJoinMethods = {} + for (const [key] of Object.entries(filter)) { + const relationPath = getRelationPath(key, mainMetadata) + // Skip filters that don't result in WHERE clauses and so don't need to be joined.. + if ( + relationPath.find( + ([, relation]) => 'isOneToMany' in relation && (relation.isOneToMany || relation.isManyToMany) + ) + ) { + continue + } + + for (let i = 0; i < relationPath.length; i++) { + const column = relationPath + .slice(0, i + 1) + .map((p) => p[0]) + .join('.') + // Skip joins on embedded entities + if ('inverseRelation' in relationPath[i][1]) { + columnJoinMethods[column] = 'leftJoinAndSelect' + } + } + } + + return columnJoinMethods +} + +export function addDirectFilters(qb: SelectQueryBuilder, filter: ColumnFilters) { const filterEntries = Object.entries(filter) + const metadata = qb.expressionMap.mainAlias.metadata - // Looks for filters that don't have a toMany relationship on their path: will be translated to WHERE clauses on this query. - const whereFilters = filterEntries.filter(([key]) => !findFirstToManyRelationship(key, mainMetadata)) + // Direct filters are those without toMany relationships on their path, and can be expressed as simple JOINs + WHERE clauses + const whereFilters = filterEntries.filter(([key]) => !findFirstToManyRelationship(key, metadata)) const orFilters = whereFilters.filter(([, value]) => value[0].comparator === '$or') const andFilters = whereFilters.filter(([, value]) => value[0].comparator === '$and') @@ -483,8 +538,18 @@ export function addFilter( }) ) } +} - // Looks filters that have a toMany relationship on their path: will be translated to EXISTS subqueries. +export function addToManySubFilters( + qb: SelectQueryBuilder, + filter: ColumnFilters, + query: PaginateQuery, + filterableColumns?: { [column: string]: (FilterOperator | FilterSuffix | FilterQuantifier)[] | true } +) { + const mainMetadata = qb.expressionMap.mainAlias.metadata + const filterEntries = Object.entries(filter) + + // Filters with toMany relationships on their path need to be expressed as EXISTS subqueries. const existsFilters = filterEntries.map(([key]) => findFirstToManyRelationship(key, mainMetadata)).filter(Boolean) // Find all the different toMany starting paths const toManyPaths = [...new Set(existsFilters.map((f) => f.path.join('.')))] @@ -498,7 +563,12 @@ export function addFilter( const toManyAlias = `_rel_${relationPath[relationPath.length - 1][0]}_${relationPath.length - 1}` // 1. Create the EXISTS subquery, starting from the toMany entity - const existsQb = qb.connection.createQueryBuilder(relation.inverseEntityMetadata.target as any, toManyAlias) + const existsMetadata = relation.inverseEntityMetadata + const existsQb = qb.connection.createQueryBuilder(existsMetadata.target as any, toManyAlias) + + if (existsMetadata.deleteDateColumn) { + //existsQb.andWhere(`"${toManyAlias}"."${existsMetadata.deleteDateColumn.databaseName}" IS NULL`) + } // 2. Add the subfilters to the EXISTS subquery const { subQuery, subFilterableColumns } = createSubFilter(query, filterableColumns, path) @@ -577,32 +647,29 @@ export function addFilter( } } - // 5. Add the EXISTS subquery to the main query - qb.andWhereExists(existsQb) - } + const quantifiers = Object.entries(filter) + .filter(([key]) => key.startsWith(path)) + .flatMap(([, multiFilter]) => multiFilter.map((f) => f.quantifier)) - const columnJoinMethods: ColumnJoinMethods = {} - for (const [key] of filterEntries) { - const relationPath = getRelationPath(key, mainMetadata) - for (let i = 0; i < relationPath.length; i++) { - const [, subRelation] = relationPath[i] - const column = relationPath - .slice(0, i + 1) - .map((p) => p[0]) - .join('.') - // Join the toMany - if ('isOneToMany' in subRelation) { - if (subRelation.isOneToOne || subRelation.isManyToOne) { - columnJoinMethods[column] = 'innerJoinAndSelect' - } else { - // Stop traversing at toMany boundaries, since those will be handled by EXISTS subqueries - break + let quantifier = FilterQuantifier.ANY + for (const q of quantifiers) { + if (q !== FilterQuantifier.ANY) { + if (quantifier !== FilterQuantifier.ANY && quantifier !== q) { + throw new Error(`Quantifier ${quantifier} and ${q} are not compatible for the same column ${path}`) } + quantifier = q } } - } - return columnJoinMethods + // 5. Add the EXISTS subquery to the main query + if (quantifier === FilterQuantifier.ANY) { + qb.andWhereExists(existsQb) + } else if (quantifier === FilterQuantifier.NONE) { + andWhereNoneExist(qb, existsQb) + } else if (quantifier === FilterQuantifier.ALL) { + andWhereAllExist(qb, existsQb) + } + } } export function createSubFilter( diff --git a/src/helper.ts b/src/helper.ts index 1d872a380..641c7d081 100644 --- a/src/helper.ts +++ b/src/helper.ts @@ -216,7 +216,7 @@ export function checkIsOneOfNestedPrimaryColumns(qb: SelectQueryBuilder } metadata = relation.inverseEntityMetadata } - return !!metadata.primaryColumns.find(col => col.propertyName === deepestProperty) + return !!metadata.primaryColumns.find((col) => col.propertyName === deepestProperty) } export function checkIsEmbedded(qb: SelectQueryBuilder, propertyPath: string): boolean { @@ -372,3 +372,39 @@ export function isNil(v: unknown): boolean { export function isNotNil(v: unknown): boolean { return !isNil(v) } + +export function andWhereNoneExist( + qb: SelectQueryBuilder, + existsQb: SelectQueryBuilder +): SelectQueryBuilder { + const [query, params] = qb['getExistsCondition'](existsQb) + return qb.andWhere(`NOT ${query}`, params) +} + +/** + * Adds a condition to the query builder that ensures all related entities match the given filter criteria. + * + * This method combines two conditions: + * 1. EXISTS(X) - There must be at least one related entity matching the criteria + * 2. NOT EXISTS(NOT X) - There must not be any related entities that don't match the criteria + * + * Together, these conditions ensure that all related entities match the filter criteria X. + * For example, when filtering pillows in a cat home, this could find homes where ALL pillows are red. + * + * If you need to include cases where there are either 0 or all entities match, use $none:$not:X instead. + * + * @param {SelectQueryBuilder} qb The main query builder instance to add the condition to. + * @param {SelectQueryBuilder} existsQb The subquery builder containing the filter criteria. + * @return {SelectQueryBuilder} The modified query builder with the combined EXISTS conditions. + */ +export function andWhereAllExist( + qb: SelectQueryBuilder, + existsQb: SelectQueryBuilder +): SelectQueryBuilder { + qb = qb.andWhereExists(existsQb) + const [query, params] = qb['getExistsCondition'](existsQb) + // The getExistsCondition clears anything that comes after WHERE, and our joining logic does not contain WHERE, + // so it should be safe to replace the first WHERE with WHERE NOT (...) and get a correct query. + const existsWhereNot = query.replace('WHERE', 'WHERE NOT (') + ')' + return qb.andWhere(`NOT ${existsWhereNot}`, params) +} diff --git a/src/paginate.spec.ts b/src/paginate.spec.ts index 96d9252cf..9a4759423 100644 --- a/src/paginate.spec.ts +++ b/src/paginate.spec.ts @@ -15,6 +15,7 @@ import { PaginateQuery } from './decorator' import { FilterComparator, FilterOperator, + FilterQuantifier, FilterSuffix, isOperator, isSuffix, @@ -61,7 +62,7 @@ describe('paginate', () => { const dbOptions: Omit, 'poolSize'> = { dropSchema: true, synchronize: true, - logging: ['error'], + logging: ['error', 'query'], entities: [ CatEntity, CatToyEntity, @@ -5162,5 +5163,134 @@ describe('paginate', () => { expect(existsSpy).toHaveBeenCalledTimes(2) }) }) + + describe('Filtering records without related entities matching filter criteria', () => { + // To be clear: "without" also means any record that simply does not have any related entities at all. + + it('should find all cats without any pillows in their home', async () => { + // This test tests absence filtering. Also asserts that the loaded relation is empty. + const config: PaginateConfig = { + sortableColumns: ['id'], + relations: ['home.pillows'], + filterableColumns: { + 'home.pillows': [FilterQuantifier.NONE], + }, + } + const query: PaginateQuery = { + filter: { + 'home.pillows': [`$none`], + }, + path: '', + } + + const result = await paginate(query, catRepo, config) + // Only cat 0 and 1 have pillows + expect(result.data.length).toBe(5) + // Cat 2 has a home with no pillows + expect(result.data[0].id).toBe(cats[2].id) + expect(result.data[0].home.pillows).toHaveLength(0) + // The rest of the cats have no homes + expect(result.data[1].id).toBe(cats[3].id) + expect(result.data[1].home).toBeNull() + expect(result.data[2].id).toBe(cats[4].id) + expect(result.data[2].home).toBeNull() + expect(result.data[3].id).toBe(cats[5].id) + expect(result.data[3].home).toBeNull() + expect(result.data[4].id).toBe(cats[6].id) + expect(result.data[4].home).toBeNull() + }) + + it('should find all cats without red pillows in their home', async () => { + // This test tests absence filtering with a single criterium. + const config: PaginateConfig = { + sortableColumns: ['id'], + filterableColumns: { + 'home.pillows.color': [FilterQuantifier.NONE], + }, + } + const query: PaginateQuery = { + filter: { + 'home.pillows.color': [`$none:red`], + }, + path: '', + } + + const result = await paginate(query, catRepo, config) + // Only cat 0 has a red pillow + expect(result.data.length).toBe(6) + expect(result.data[0].id).toBe(cats[1].id) + expect(result.data[1].id).toBe(cats[2].id) + expect(result.data[2].id).toBe(cats[3].id) + expect(result.data[3].id).toBe(cats[4].id) + expect(result.data[4].id).toBe(cats[5].id) + expect(result.data[5].id).toBe(cats[6].id) + }) + + it('should find all cats without red or teal pillows in their home', async () => { + // This test tests absence filtering with multiple criteria. + const config: PaginateConfig = { + sortableColumns: ['id'], + filterableColumns: { + 'home.pillows.color': [FilterQuantifier.NONE], + }, + } + const query: PaginateQuery = { + filter: { + 'home.pillows.color': [`$none:red`, '$or:teal'], + }, + path: '', + } + + const result = await paginate(query, catRepo, config) + // Cat 0 has a red pillow, and cat 1 has a teal pillow. + expect(result.data.length).toBe(5) + expect(result.data[0].id).toBe(cats[2].id) + expect(result.data[1].id).toBe(cats[3].id) + expect(result.data[2].id).toBe(cats[4].id) + expect(result.data[3].id).toBe(cats[5].id) + expect(result.data[4].id).toBe(cats[6].id) + }) + + describe('Advanced quantifier combinatorics', () => { + // None of these have been implemented yet, feel free to PR :innocent: + + it('should error with multiple different quantifiers on the same column', async () => { + // This test tests absence filtering with multiple criteria. + const config: PaginateConfig = { + sortableColumns: ['id'], + filterableColumns: { + 'home.pillows.color': [FilterQuantifier.NONE, FilterQuantifier.ALL], + }, + } + const query: PaginateQuery = { + filter: { + 'home.pillows.color': [`$none:red`, `$all:blue`], + }, + path: '', + } + await expect(paginate(query, catRepo, config)).rejects.toBeDefined() + }) + + it('should error with multiple different quantifiers on the same relationship', async () => { + // This test tests absence filtering with a multiple criterium. + const config: PaginateConfig = { + sortableColumns: ['id'], + filterableColumns: { + 'home.pillows.color': [FilterQuantifier.NONE], + 'home.pillows.brand.name': [FilterQuantifier.ALL, FilterOperator.ILIKE], + }, + } + const query: PaginateQuery = { + filter: { + 'home.pillows.color': [`$none:red`], + 'home.pillows.brand.name': [`$all:$ilike:purr`], + }, + path: '', + } + + await expect(paginate(query, catRepo, config)).rejects.toBeDefined() + }) + }) + }) }) }) diff --git a/src/paginate.ts b/src/paginate.ts index 98c467f95..6aef67e9b 100644 --- a/src/paginate.ts +++ b/src/paginate.ts @@ -12,7 +12,7 @@ import { } from 'typeorm' import { WherePredicateOperator } from 'typeorm/query-builder/WhereClause' import { PaginateQuery } from './decorator' -import { addFilter, FilterOperator, FilterSuffix } from './filter' +import { addFilter, FilterOperator, FilterQuantifier, FilterSuffix } from './filter' import { checkIsEmbedded, checkIsRelation, @@ -91,7 +91,7 @@ export interface PaginateConfig { defaultSortBy?: SortBy defaultLimit?: number where?: FindOptionsWhere | FindOptionsWhere[] - filterableColumns?: Partial> + filterableColumns?: Partial> loadEagerRelations?: boolean withDeleted?: boolean allowWithDeletedInQuery?: boolean @@ -1024,7 +1024,7 @@ export function addRelationsFromSchema( queryBuilder: SelectQueryBuilder, schema: RelationSchema, joinMethods: Partial>, - defaultJoinMethod: 'leftJoin' | 'innerJoin' | 'leftJoinAndSelect' | 'innerJoinAndSelect' = 'leftJoinAndSelect', + defaultJoinMethod: 'leftJoin' | 'innerJoin' | 'leftJoinAndSelect' | 'innerJoinAndSelect' = 'leftJoinAndSelect' ): void { const createQueryBuilderRelations = ( prefix: string, From 994da4798b3e67fa22817c229aca789cb6f83a95 Mon Sep 17 00:00:00 2001 From: Robin De Schepper Date: Sat, 25 Oct 2025 22:33:35 +0200 Subject: [PATCH 7/9] fixed spec and format --- src/filter.ts | 13 ++++++----- src/paginate.spec.ts | 53 +++++++++++++------------------------------- 2 files changed, 23 insertions(+), 43 deletions(-) diff --git a/src/filter.ts b/src/filter.ts index 2a117af6b..bd5cdc089 100644 --- a/src/filter.ts +++ b/src/filter.ts @@ -256,11 +256,12 @@ export function parseFilterToken(raw?: string): FilterToken | null { } if (notValue.length) { - token.value = !raw.includes(OPERAND_SEPARATOR) - ? // things like `$null`, `$none`, and `$any`, have no token value - undefined - : // otherwise, remove the operators and separators from the raw string to obtain the token value - raw.replace(`${notValue.join(OPERAND_SEPARATOR)}${OPERAND_SEPARATOR}`, '') + token.value = + !raw.includes(OPERAND_SEPARATOR) || token.operator === FilterOperator.NULL + ? // things like `$null`, `$none`, and `$any`, have no token value + undefined + : // otherwise, remove the operators and separators from the raw string to obtain the token value + raw.replace(`${notValue.join(OPERAND_SEPARATOR)}${OPERAND_SEPARATOR}`, '') } return token @@ -506,7 +507,7 @@ export function addFilter( .join('.') // Skip joins on embedded entities if ('inverseRelation' in relationPath[i][1]) { - columnJoinMethods[column] = 'leftJoinAndSelect' + columnJoinMethods[column] = 'innerJoinAndSelect' } } } diff --git a/src/paginate.spec.ts b/src/paginate.spec.ts index 9a4759423..3670ee69e 100644 --- a/src/paginate.spec.ts +++ b/src/paginate.spec.ts @@ -62,7 +62,7 @@ describe('paginate', () => { const dbOptions: Omit, 'poolSize'> = { dropSchema: true, synchronize: true, - logging: ['error', 'query'], + logging: ['error'], entities: [ CatEntity, CatToyEntity, @@ -1381,17 +1381,20 @@ describe('paginate', () => { const cat1 = clone(cats[0]) const cat2 = clone(cats[1]) const catToys1 = clone(catToysWithoutShop[0]) - const catToys2 = clone(catToysWithoutShop[2]) - const catToys3 = clone(catToysWithoutShop[3]) + const catToys2 = clone(catToysWithoutShop[1]) + const catToys3 = clone(catToysWithoutShop[2]) + const catToys4 = clone(catToysWithoutShop[3]) delete catToys1.cat delete catToys2.cat delete catToys3.cat - cat1.toys = [catToys1, catToys2] - cat2.toys = [catToys3] + delete catToys4.cat + cat1.toys = [catToys3, catToys2, catToys1] + cat2.toys = [catToys4] expect(result.meta.filter).toStrictEqual({ 'toys.name': '$not:Stuffed Mouse', }) + console.log(result.data[0].toys, result.data[1].toys) expect(result.data).toStrictEqual([cat1, cat2]) expect(result.links.current).toBe('?page=1&limit=20&sortBy=id:ASC&filter.toys.name=$not:Stuffed Mouse') }) @@ -2441,7 +2444,7 @@ describe('paginate', () => { tokens: { comparator, operator: '$null', suffix: '$not', value: undefined }, }, ])('should get filter tokens for "$string"', ({ string, tokens }) => { - expect(parseFilterToken(string)).toStrictEqual(tokens) + expect(parseFilterToken(string)).toStrictEqual({ quantifier: '$any', ...tokens }) }) } @@ -2963,11 +2966,13 @@ describe('paginate', () => { const cat = clone(cats[1]) const catHomesClone = clone(catHomes[1]) - const catHomePillowsClone = clone(catHomePillows[3]) - delete catHomePillowsClone.home + const catHomePillowsClone = clone(catHomePillows.slice(3, 6)) + catHomePillowsClone.forEach((pillow) => { + delete pillow.home + }) catHomesClone.countCat = cats.filter((cat) => cat.id === catHomesClone.cat.id).length - catHomesClone.pillows = [catHomePillowsClone] + catHomesClone.pillows = catHomePillowsClone cat.home = catHomesClone delete cat.home.cat @@ -3358,32 +3363,6 @@ describe('paginate', () => { }) }) - describe('$null operator', () => { - it('should find cats without any toys when applied on PK', async () => { - const config: PaginateConfig = { - relations: ['toys'], - sortableColumns: ['id', 'toys.id'], - filterableColumns: { - 'toys.id': [FilterOperator.NULL], - }, - } - const query: PaginateQuery = { - filter: { - // Null-filtering a relationship's PK should check - // for absence of the relationship - 'toys.id': '$null', - }, - path: '', - } - - const result = await paginate(query, catRepo, config) - // Should find only cats without toys. - expect(result.data.every((cat) => cat.toys.length === 0)).toBe(true) - // Should find 5 cats without toys. - expect(result.data.length).toBe(5) - }) - }) - describe('should correctly handle number column filter', () => { it('with $eq operator and valid number', async () => { const config: PaginateConfig = { @@ -4157,7 +4136,7 @@ describe('paginate', () => { const result = await paginate(query, catRepo, config) - const sortedCats = cats.sort((a, b) => a.weightChange - b.weightChange) + const sortedCats = [...cats].sort((a, b) => a.weightChange - b.weightChange) expect(result.data).toEqual(sortedCats) expect(result.links.previous).toBe('?limit=20&sortBy=weightChange:DESC&cursor=M99999999997X0000') // weightChange=-3.00 DESC (Shadow) -> (M + 10^11 - 3) + (X + PAD(0, 4, '0')) expect(result.links.next).toBe('?limit=20&sortBy=weightChange:ASC&cursor=V99999999995V7500') // weightChange=5.25 ASC (Garfield) -> (V + 10^11 - 5) + (V + 10^4 - 2500) @@ -4175,7 +4154,7 @@ describe('paginate', () => { const result = await paginate(query, catRepo, config) - const sortedCats = cats.sort((a, b) => b.weightChange - a.weightChange) + const sortedCats = [...cats].sort((a, b) => b.weightChange - a.weightChange) expect(result.data).toEqual(sortedCats) expect(result.links.previous).toBe('?limit=20&sortBy=weightChange:ASC&cursor=V99999999995V7500') // weightChange=5.25 ASC (Garfield) -> (V + 10^11 - 5) + (V + 10^4 - 2500) expect(result.links.next).toBe('?limit=20&sortBy=weightChange:DESC&cursor=M99999999997X0000') // weightChange=-3.00 DESC (Shadow) -> (M + 10^11 - 3) + (X + LPAD(0, 4, '0')) From be8708db85654a350c76e68fba9ddcb926eade68 Mon Sep 17 00:00:00 2001 From: Robin De Schepper Date: Mon, 27 Oct 2025 15:21:08 +0100 Subject: [PATCH 8/9] updated docs --- README.md | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/README.md b/README.md index 27b40cefd..7463bef8f 100644 --- a/README.md +++ b/README.md @@ -516,6 +516,48 @@ const config: PaginateConfig = { const result = await paginate(query, catRepo, config) ``` +## Usage with to-many relationships + +You can filter parents by conditions on their to-many relations (one-to-many or many-to-many) using quantifiers. +Quantifiers define how many related rows must satisfy the condition: + +- `$any` (default): at least one related row matches the condition +- `$all`: all related rows match the condition +- `$none`: no related rows match the condition + +### Examples +Assume `CatEntity` has a one‑to‑many relation `toys: CatToyEntity[]` where `CatToyEntity` has a string column `name`. + +- At least one toy named exactly "Ball": + + ```url + GET /cats?filter.toys.name=$any:$eq:Ball + ``` + +- At least one toy whose name contains "red" (case-insensitive): + + ```url + GET /cats?filter.toys.name=$any:$ilike:red + ``` + +- All toys must have names that start with "Chew": + + ```url + GET /cats?filter.toys.name=$all:$sw:Chew + ``` + +- No toys named "Squeaky", including cats without any toys: + + ```url + GET /cats?filter.toys.name=$none:$eq:Squeaky + ``` + +- One or more toys not named "Squeaky": + + ```url + GET /cats?filter.toys.name=$any:$not:$eq:Squeaky + ``` + ## Usage with Eager Loading Eager loading should work with TypeORM's eager property out of the box: From 89e0f73263d2efdf7bdf48c2840441d9633ff595 Mon Sep 17 00:00:00 2001 From: Robin De Schepper Date: Fri, 7 Nov 2025 09:33:58 +0100 Subject: [PATCH 9/9] fixed mariadb quotes in tomany filters --- src/filter.ts | 7 +++++-- src/helper.ts | 2 +- src/paginate.ts | 4 ++-- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/filter.ts b/src/filter.ts index bd5cdc089..1e4977265 100644 --- a/src/filter.ts +++ b/src/filter.ts @@ -34,6 +34,7 @@ import { isISODate, JoinMethod, mergeRelationSchema, + quoteColumn, } from './helper' import { EmbeddedMetadata } from 'typeorm/metadata/EmbeddedMetadata' import { RelationMetadata } from 'typeorm/metadata/RelationMetadata' @@ -547,6 +548,8 @@ export function addToManySubFilters( query: PaginateQuery, filterableColumns?: { [column: string]: (FilterOperator | FilterSuffix | FilterQuantifier)[] | true } ) { + const dbType = qb.connection.options.type + const quote = (column: string) => quoteColumn(column, ['mysql', 'mariadb'].includes(dbType)) const mainMetadata = qb.expressionMap.mainAlias.metadata const filterEntries = Object.entries(filter) @@ -613,7 +616,7 @@ export function addToManySubFilters( .map((jc) => { const fk = jc.databaseName const pk = jc.referencedColumn.databaseName - return `"${childAlias}"."${fk}" = "${parentAlias}"."${pk}"` + return `${quote(childAlias)}.${quote(fk)} = ${quote(parentAlias)}.${quote(pk)}` }) .join(' AND ') @@ -643,7 +646,7 @@ export function addToManySubFilters( } // Correlation - existsQb.andWhere(`"${fkAlias}"."${fkColumn}" = "${pkAlias}"."${pkColumn}"`) + existsQb.andWhere(`${quote(fkAlias)}.${quote(fkColumn)} = ${quote(pkAlias)}.${quote(pkColumn)}`) } } } diff --git a/src/helper.ts b/src/helper.ts index 641c7d081..84c230a8c 100644 --- a/src/helper.ts +++ b/src/helper.ts @@ -361,7 +361,7 @@ export function isDateColumnType(type: any): boolean { return dateTypes.includes(type) } -export function quoteVirtualColumn(columnName: string, isMySqlOrMariaDb: boolean): string { +export function quoteColumn(columnName: string, isMySqlOrMariaDb: boolean): string { return isMySqlOrMariaDb ? `\`${columnName}\`` : `"${columnName}"` } diff --git a/src/paginate.ts b/src/paginate.ts index 6aef67e9b..6d107c442 100644 --- a/src/paginate.ts +++ b/src/paginate.ts @@ -36,7 +36,7 @@ import { mergeRelationSchema, Order, positiveNumberOrDefault, - quoteVirtualColumn, + quoteColumn, RelationSchema, RelationSchemaInput, SortBy, @@ -692,7 +692,7 @@ export async function paginate( let alias = fixColumnAlias(columnProperties, queryBuilder.alias, isRelation, isVirtualProperty, isEmbedded) if (isVirtualProperty) { - alias = quoteVirtualColumn(alias, isMySqlOrMariaDb) + alias = quoteColumn(alias, isMySqlOrMariaDb) } if (isMySqlOrMariaDb) {