diff --git a/packages/cubejs-backend-shared/src/env.ts b/packages/cubejs-backend-shared/src/env.ts index 3b0a155f880f3..19f589f5f0223 100644 --- a/packages/cubejs-backend-shared/src/env.ts +++ b/packages/cubejs-backend-shared/src/env.ts @@ -649,6 +649,13 @@ const variables: Record any> = { .default(8192) .asInt(), + /** + * Max number of elements + */ + usedPreAggregationCacheMaxCount: (): number => get('CUBEJS_USED_PRE_AGG_CACHE_MAX_COUNT') + .default(8192) + .asInt(), + /** * Max number of elements */ diff --git a/packages/cubejs-query-orchestrator/src/orchestrator/PreAggregations.ts b/packages/cubejs-query-orchestrator/src/orchestrator/PreAggregations.ts index 30a45ff1da9d9..c8a6722cc9049 100644 --- a/packages/cubejs-query-orchestrator/src/orchestrator/PreAggregations.ts +++ b/packages/cubejs-query-orchestrator/src/orchestrator/PreAggregations.ts @@ -242,11 +242,11 @@ type PreAggregationQueryBody = QueryBody & { }; export class PreAggregations { - public options: PreAggregationsOptions; + public readonly options: PreAggregationsOptions; - public externalDriverFactory: DriverFactory; + public readonly externalDriverFactory: DriverFactory; - public structureVersionPersistTime: any; + public readonly structureVersionPersistTime: any; private readonly touchTablePersistTime: number; @@ -260,6 +260,8 @@ export class PreAggregations { private readonly queue: Record = {}; + private readonly usedCache: LRUCache; + private readonly touchCache: LRUCache; public constructor( @@ -277,6 +279,25 @@ export class PreAggregations { this.dropPreAggregationsWithoutTouch = options.dropPreAggregationsWithoutTouch || getEnv('dropPreAggregationsWithoutTouch'); this.usedTablePersistTime = options.usedTablePersistTime || getEnv('dbQueryTimeout'); this.externalRefresh = options.externalRefresh; + + /** + * Comment for both caches: used and touch: + * + * Memory usage: By default, it defines max as 8096 keys, + * let's assume that avg key size is 64 symbols, it's around 100 bytes + * with V8's internal stuff: + * + * 100 x 8096 = 809 bytes, max 1Mb in memory. + * + * However, this is a theoretical limit. In practice, even a couple of thousand + * is a very large number + */ + this.usedCache = new LRUCache({ + max: getEnv('usedPreAggregationCacheMaxCount'), + ttl: Math.round(this.usedTablePersistTime / 2) * 1000, + allowStale: false, + updateAgeOnGet: false + }); this.touchCache = new LRUCache({ max: getEnv('touchPreAggregationCacheMaxCount'), ttl: getEnv('touchPreAggregationCacheMaxAge') * 1000, @@ -301,11 +322,23 @@ export class PreAggregations { } public async addTableUsed(tableName: string): Promise { - await this.queryCache.getCacheDriver().set( - this.tablesUsedRedisKey(tableName), - true, - this.usedTablePersistTime - ); + if (this.usedCache.has(tableName)) { + return; + } + + try { + this.usedCache.set(tableName, true); + + await this.queryCache.getCacheDriver().set( + this.tablesUsedRedisKey(tableName), + true, + this.usedTablePersistTime + ); + } catch (e: unknown) { + this.usedCache.delete(tableName); + + throw e; + } } public async tablesUsed() { diff --git a/packages/cubejs-query-orchestrator/src/orchestrator/QueryCache.ts b/packages/cubejs-query-orchestrator/src/orchestrator/QueryCache.ts index 3c07ac545ed5d..f3b6fbccce462 100644 --- a/packages/cubejs-query-orchestrator/src/orchestrator/QueryCache.ts +++ b/packages/cubejs-query-orchestrator/src/orchestrator/QueryCache.ts @@ -130,7 +130,7 @@ export class QueryCache { protected memoryCache: LRUCache; public constructor( - protected readonly redisPrefix: string, + protected readonly cachePrefix: string, protected readonly driverFactory: DriverFactoryByDataSource, protected readonly logger: any, public readonly options: QueryCacheOptions @@ -165,11 +165,7 @@ export class QueryCache { } public getKey(catalog: string, key: string): string { - if (this.cacheDriver instanceof CubeStoreCacheDriver) { - return `${this.redisPrefix}#${catalog}:${key}`; - } else { - return `${catalog}_${this.redisPrefix}_${key}`; - } + return `${this.cachePrefix}#${catalog}:${key}`; } /** @@ -469,7 +465,7 @@ export class QueryCache { const queueOptions = await this.options.queueOptions(dataSource); if (!this.queue[dataSource]) { this.queue[dataSource] = QueryCache.createQueue( - `SQL_QUERY_${this.redisPrefix}_${dataSource}`, + `SQL_QUERY_${this.cachePrefix}_${dataSource}`, () => this.driverFactory(dataSource), (client, req) => { this.logger('Executing SQL', { ...req }); @@ -537,7 +533,7 @@ export class QueryCache { public getExternalQueue() { if (!this.externalQueue) { this.externalQueue = QueryCache.createQueue( - `SQL_QUERY_EXT_${this.redisPrefix}`, + `SQL_QUERY_EXT_${this.cachePrefix}`, this.options.externalDriverFactory, (client, q) => { this.logger('Executing SQL', {