From 7038fd88e3cd2aefafed558cb545805f9d605382 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Fri, 4 Apr 2025 11:35:53 +0300 Subject: [PATCH 1/2] feat(server-core): Make OrchestratorStorage LRUCache --- .../src/core/OrchestratorStorage.ts | 39 +++++++------------ 1 file changed, 13 insertions(+), 26 deletions(-) diff --git a/packages/cubejs-server-core/src/core/OrchestratorStorage.ts b/packages/cubejs-server-core/src/core/OrchestratorStorage.ts index deb51e40971c6..cc1f6de078b00 100644 --- a/packages/cubejs-server-core/src/core/OrchestratorStorage.ts +++ b/packages/cubejs-server-core/src/core/OrchestratorStorage.ts @@ -1,7 +1,16 @@ +import { LRUCache } from 'lru-cache'; import type { OrchestratorApi } from './OrchestratorApi'; export class OrchestratorStorage { - protected readonly storage: Map = new Map(); + protected readonly storage: LRUCache; + + public constructor(options: { compilerCacheSize?: number, maxCompilerCacheKeepAlive?: number, updateCompilerCacheKeepAlive?: boolean } = { compilerCacheSize: 100 }) { + this.storage = new LRUCache({ + max: options.compilerCacheSize, + ttl: options.maxCompilerCacheKeepAlive, + updateAgeOnGet: options.updateCompilerCacheKeepAlive + }); + } public has(orchestratorId: string) { return this.storage.has(orchestratorId); @@ -20,37 +29,15 @@ export class OrchestratorStorage { } public async testConnections() { - const result = []; - - // eslint-disable-next-line no-restricted-syntax - for (const orchestratorApi of this.storage.values()) { - result.push(orchestratorApi.testConnection()); - } - - return Promise.all(result); + return Promise.all([...this.storage.values()].map(api => api.testConnection())); } public async testOrchestratorConnections() { - const result = []; - - // eslint-disable-next-line no-restricted-syntax - for (const orchestratorApi of this.storage.values()) { - result.push(orchestratorApi.testOrchestratorConnections()); - } - - return Promise.all(result); + return Promise.all([...this.storage.values()].map(api => api.testOrchestratorConnections())); } public async releaseConnections() { - const result = []; - - // eslint-disable-next-line no-restricted-syntax - for (const orchestratorApi of this.storage.values()) { - result.push(orchestratorApi.release()); - } - - await Promise.all(result); - + await Promise.all([...this.storage.values()].map(api => api.release())); this.storage.clear(); } } From 259b85b6c43adf54faa07bf140a2ecaf2eb11c78 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Thu, 27 Mar 2025 13:26:42 +0200 Subject: [PATCH 2/2] get rid of ramda # Conflicts: # packages/cubejs-server-core/src/core/CompilerApi.js --- packages/cubejs-server-core/src/core/CompilerApi.js | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/packages/cubejs-server-core/src/core/CompilerApi.js b/packages/cubejs-server-core/src/core/CompilerApi.js index 0a78e27edf2b2..f92ae16e30727 100644 --- a/packages/cubejs-server-core/src/core/CompilerApi.js +++ b/packages/cubejs-server-core/src/core/CompilerApi.js @@ -1,5 +1,4 @@ import crypto from 'crypto'; -import R from 'ramda'; import { createQuery, compile, queryClass, PreAggregations, QueryFactory } from '@cubejs-backend/schema-compiler'; import { v4 as uuidv4, parse as uuidParse } from 'uuid'; import { LRUCache } from 'lru-cache'; @@ -128,9 +127,9 @@ export class CompilerApi { async createQueryFactory(compilers) { const { cubeEvaluator } = compilers; - const cubeToQueryClass = R.fromPairs( + const cubeToQueryClass = Object.fromEntries( await Promise.all( - cubeEvaluator.cubeNames().map(async cube => { + cubeEvaluator.cubeNames().map(async (cube) => { const dataSource = cubeEvaluator.cubeFromPath(cube).dataSource ?? 'default'; const dbType = await this.getDbType(dataSource); const dialectClass = this.getDialectClass(dataSource, dbType); @@ -146,7 +145,7 @@ export class CompilerApi { } getDialectClass(dataSource = 'default', dbType) { - return this.dialectClass && this.dialectClass({ dataSource, dbType }); + return this.dialectClass?.({ dataSource, dbType }); } async getSqlGenerator(query, dataSource) { @@ -192,8 +191,8 @@ export class CompilerApi { external: sqlGenerator.externalPreAggregationQuery(), sql: sqlGenerator.buildSqlAndParams(exportAnnotatedSql), lambdaQueries: sqlGenerator.buildLambdaQuery(), - timeDimensionAlias: sqlGenerator.timeDimensions[0] && sqlGenerator.timeDimensions[0].unescapedAliasName(), - timeDimensionField: sqlGenerator.timeDimensions[0] && sqlGenerator.timeDimensions[0].dimension, + timeDimensionAlias: sqlGenerator.timeDimensions[0]?.unescapedAliasName(), + timeDimensionField: sqlGenerator.timeDimensions[0]?.dimension, order: sqlGenerator.order, cacheKeyQueries: sqlGenerator.cacheKeyQueries(), preAggregations: sqlGenerator.preAggregations.preAggregationsDescription(), @@ -227,7 +226,7 @@ export class CompilerApi { } roleMeetsConditions(evaluatedConditions) { - if (evaluatedConditions && evaluatedConditions.length) { + if (evaluatedConditions?.length) { return evaluatedConditions.reduce((a, b) => { if (typeof b !== 'boolean') { throw new Error(`Access policy condition must return boolean, got ${JSON.stringify(b)}`);