diff --git a/packages/cubejs-backend-native/.gitignore b/packages/cubejs-backend-native/.gitignore index 46a96f32f5b0c..67cbee8dc09f3 100644 --- a/packages/cubejs-backend-native/.gitignore +++ b/packages/cubejs-backend-native/.gitignore @@ -6,3 +6,5 @@ index.node **/node_modules **/.DS_Store npm-debug.log +benchmarks/result.txt +test/__pycache__/ \ No newline at end of file diff --git a/packages/cubejs-backend-native/benchmarks/fixtures/config-async.py b/packages/cubejs-backend-native/benchmarks/fixtures/config-async.py new file mode 100644 index 0000000000000..a14a6608d9448 --- /dev/null +++ b/packages/cubejs-backend-native/benchmarks/fixtures/config-async.py @@ -0,0 +1,112 @@ +from cube import config, file_repository + +config.schema_path = "models" +config.pg_sql_port = 5555 +config.telemetry = False + + +@config +async def query_rewrite(query, ctx): + # Removed print statements for benchmarking + return query + + +@config +async def check_auth(req, authorization): + # Removed print statements for benchmarking + return { + "security_context": {"sub": "1234567890", "iat": 1516239022, "user_id": 42}, + "ignoredField": "should not be visible", + } + + +@config('extend_context') +async def extend_context(req): + # Removed print statements for benchmarking + if "securityContext" not in req: + return { + "security_context": { + "error": "missing", + } + } + + req["securityContext"]["extended_by_config"] = True + + return { + "security_context": req["securityContext"], + } + + +@config +async def repository_factory(ctx): + # Removed print statements for benchmarking + return file_repository(ctx["securityContext"]["schemaPath"]) + + +@config +async def context_to_api_scopes(): + # Removed print statements for benchmarking + return ["meta", "data", "jobs"] + + +@config +async def scheduled_refresh_time_zones(ctx): + # Removed print statements for benchmarking + return ["Europe/Kyiv", "Antarctica/Troll", "Australia/Sydney"] + + +@config +async def scheduled_refresh_contexts(ctx): + # Removed print statements for benchmarking + return [ + { + "securityContext": { + "appid": 'test1', "u": { "prop1": "value1" } + } + }, + { + "securityContext": { + "appid": 'test2', "u": { "prop1": "value2" } + } + }, + { + "securityContext": { + "appid": 'test3', "u": { "prop1": "value3" } + } + }, + ] + + +@config +async def schema_version(ctx): + # Removed print statements for benchmarking + return "1" + + +@config +async def pre_aggregations_schema(ctx): + # Removed print statements for benchmarking + return "schema" + + +@config +async def logger(msg, params): + # Removed print statements for benchmarking + pass + + +@config +async def context_to_roles(ctx): + # Removed print statements for benchmarking + return [ + "admin", + ] + + +@config +async def context_to_groups(ctx): + # Removed print statements for benchmarking + return [ + "dev", + "analytics", + ] \ No newline at end of file diff --git a/packages/cubejs-backend-native/benchmarks/fixtures/config.py b/packages/cubejs-backend-native/benchmarks/fixtures/config.py new file mode 100644 index 0000000000000..b00cce6560e3e --- /dev/null +++ b/packages/cubejs-backend-native/benchmarks/fixtures/config.py @@ -0,0 +1,112 @@ +from cube import config, file_repository + +config.schema_path = "models" +config.pg_sql_port = 5555 +config.telemetry = False + + +@config +def query_rewrite(query, ctx): + # Removed print statements for benchmarking + return query + + +@config +def check_auth(req, authorization): + # Removed print statements for benchmarking + return { + "security_context": {"sub": "1234567890", "iat": 1516239022, "user_id": 42}, + "ignoredField": "should not be visible", + } + + +@config('extend_context') +def extend_context(req): + # Removed print statements for benchmarking + if "securityContext" not in req: + return { + "security_context": { + "error": "missing", + } + } + + req["securityContext"]["extended_by_config"] = True + + return { + "security_context": req["securityContext"], + } + + +@config +def repository_factory(ctx): + # Removed print statements for benchmarking + return file_repository(ctx["securityContext"]["schemaPath"]) + + +@config +def context_to_api_scopes(): + # Removed print statements for benchmarking + return ["meta", "data", "jobs"] + + +@config +def scheduled_refresh_time_zones(ctx): + # Removed print statements for benchmarking + return ["Europe/Kyiv", "Antarctica/Troll", "Australia/Sydney"] + + +@config +def scheduled_refresh_contexts(ctx): + # Removed print statements for benchmarking + return [ + { + "securityContext": { + "appid": 'test1', "u": { "prop1": "value1" } + } + }, + { + "securityContext": { + "appid": 'test2', "u": { "prop1": "value2" } + } + }, + { + "securityContext": { + "appid": 'test3', "u": { "prop1": "value3" } + } + }, + ] + + +@config +def schema_version(ctx): + # Removed print statements for benchmarking + return "1" + + +@config +def pre_aggregations_schema(ctx): + # Removed print statements for benchmarking + return "schema" + + +@config +def logger(msg, params): + # Removed print statements for benchmarking + pass + + +@config +def context_to_roles(ctx): + # Removed print statements for benchmarking + return [ + "admin", + ] + + +@config +def context_to_groups(ctx): + # Removed print statements for benchmarking + return [ + "dev", + "analytics", + ] \ No newline at end of file diff --git a/packages/cubejs-backend-native/benchmarks/python-config.bench.ts b/packages/cubejs-backend-native/benchmarks/python-config.bench.ts new file mode 100644 index 0000000000000..5c648123bcaf7 --- /dev/null +++ b/packages/cubejs-backend-native/benchmarks/python-config.bench.ts @@ -0,0 +1,122 @@ +import fs from 'fs/promises'; +import path from 'path'; +import { benchmarkSuite } from 'jest-bench'; + +import * as native from '../js'; +import { PyConfiguration } from '../js'; + +async function loadConfigurationFile(fileName: string): Promise { + const fullFileName = path.join(process.cwd(), 'benchmarks', 'fixtures', fileName); + const content = await fs.readFile(fullFileName, 'utf8'); + + return native.pythonLoadConfig(content, { + fileName: fullFileName + }); +} + +// Global variables to hold loaded configs and test data +let configPy: PyConfiguration; +let configAsyncPy: PyConfiguration; +let configContent: string; +let configAsyncContent: string; + +describe('Python Configuration Loading', () => { + beforeAll(async () => { + // Load file contents once for all benchmarks + const configPath = path.join(process.cwd(), 'benchmarks', 'fixtures', 'config.py'); + const configAsyncPath = path.join(process.cwd(), 'benchmarks', 'fixtures', 'config-async.py'); + const oldConfigPath = path.join(process.cwd(), 'test', 'old-config.py'); + + [configContent, configAsyncContent] = await Promise.all([ + fs.readFile(configPath, 'utf8'), + fs.readFile(configAsyncPath, 'utf8'), + ]); + + // Pre-load configurations for function benchmarks + configPy = await loadConfigurationFile('config.py'); + configAsyncPy = await loadConfigurationFile('config-async.py'); + }); + + benchmarkSuite('Config Loading', { + 'Load config.py': async () => { + const fullFileName = path.join(process.cwd(), 'benchmarks', 'fixtures', 'config.py'); + await native.pythonLoadConfig(configContent, { fileName: fullFileName }); + }, + + 'Load config-async.py': async () => { + const fullFileName = path.join(process.cwd(), 'benchmarks', 'fixtures', 'config-async.py'); + await native.pythonLoadConfig(configAsyncContent, { fileName: fullFileName }); + } + }); + + benchmarkSuite('Data Conversion Performance', { + 'Small payload (3 fields)': async () => { + const smallPayload = { simple_string: 'test', number: 42, boolean: true }; + await configPy.queryRewrite!(smallPayload, {}); + }, + + 'Medium payload (100 users)': async () => { + const mediumPayload = { + users: Array.from({ length: 100 }, (_, i) => ({ + id: i, name: `User ${i}`, active: i % 2 === 0 + })) + }; + await configPy.queryRewrite!(mediumPayload, {}); + }, + + 'Large payload (1000 items)': async () => { + const largePayload = { + data: Array.from({ length: 1000 }, (_, i) => ({ + id: i, value: `Value ${i}`, nested: { prop: i * 2 } + })) + }; + await configPy.queryRewrite!(largePayload, {}); + } + }); + + benchmarkSuite('Function Execution', { + 'checkAuth - sync version (sequential)': async () => { + await configPy.checkAuth!({ requestId: 'sync-bench' }, 'SYNC_TOKEN'); + }, + + // It should help to identify any potential issues with GIL + 'checkAuth - sync version (parallel 50x)': async () => { + await Promise.all( + Array.from({ length: 50 }, () => configPy.checkAuth!({ requestId: 'sync-bench' }, 'SYNC_TOKEN')) + ); + }, + + 'checkAuth - async version (sequential)': async () => { + await configAsyncPy.checkAuth!({ requestId: 'async-bench' }, 'ASYNC_TOKEN'); + }, + + // It should help to identify any potential issues with GIL + 'checkAuth - async version (parallel 50x)': async () => { + await Promise.all( + Array.from({ length: 50 }, () => configAsyncPy.checkAuth!({ requestId: 'async-bench' }, 'ASYNC_TOKEN')) + ); + }, + + 'extendContext - sync version': async () => { + await configPy.extendContext!({ + securityContext: { sub: '1234567890', iat: 1516239022, user_id: 42 } + }); + }, + + 'extendContext - async version (sequential)': async () => { + await configAsyncPy.extendContext!({ + securityContext: { sub: '1234567890', iat: 1516239022, user_id: 42 } + }); + }, + + 'queryRewrite - sync version': async () => { + const testQuery = { str: 'string', int_number: 1, bool_true: true }; + await configPy.queryRewrite!(testQuery, {}); + }, + + 'queryRewrite - async version (sequential)': async () => { + const testQuery = { str: 'string', int_number: 1, bool_true: true }; + await configAsyncPy.queryRewrite!(testQuery, {}); + }, + }); +}); \ No newline at end of file diff --git a/packages/cubejs-backend-native/jest-bench.config.js b/packages/cubejs-backend-native/jest-bench.config.js new file mode 100644 index 0000000000000..aab03d4eaa2b5 --- /dev/null +++ b/packages/cubejs-backend-native/jest-bench.config.js @@ -0,0 +1,22 @@ +const base = require('../../jest.base.config'); + +/** @type {import('jest').Config} */ +module.exports = { + ...base, + rootDir: '.', + testEnvironment: 'jest-bench/environment', + testEnvironmentOptions: { + testEnvironment: 'node', + testEnvironmentOptions: {} + }, + // Include default reporter for error reporting alongside jest-bench reporter + reporters: ['default', 'jest-bench/reporter'], + // Pick up *.bench.ts files + testRegex: '\\.bench\\.(ts|tsx|js)$', + roots: [ + '/dist/benchmarks/' + ], + snapshotResolver: '/test/snapshotResolver.js', + // Coverage is not needed for benchmarks + collectCoverage: false +}; diff --git a/packages/cubejs-backend-native/package.json b/packages/cubejs-backend-native/package.json index 4185987ecb868..a3ee1c6fc5359 100644 --- a/packages/cubejs-backend-native/package.json +++ b/packages/cubejs-backend-native/package.json @@ -24,6 +24,7 @@ "unit": "jest --forceExit", "test:unit": "yarn run unit", "test:cargo": "cargo test", + "bench": "jest --config jest-bench.config.js --forceExit", "lint": "eslint test/ js/ --ext .ts", "lint:fix": "eslint --fix test/ js/ --ext .ts" }, @@ -39,6 +40,7 @@ "@types/node": "^20", "cargo-cp-artifact": "^0.1.9", "jest": "^29", + "jest-bench": "^29", "pg": "^8.11.3", "typescript": "~5.2.2", "uuid": "^8.3.2" diff --git a/packages/cubejs-backend-native/test/jinja.test.ts b/packages/cubejs-backend-native/test/jinja.test.ts index 88b1505c933b2..ddc34541ca89b 100644 --- a/packages/cubejs-backend-native/test/jinja.test.ts +++ b/packages/cubejs-backend-native/test/jinja.test.ts @@ -9,8 +9,6 @@ type InitJinjaFn = () => Promise<{ }>; const suite = native.isFallbackBuild() ? xdescribe : describe; -// TODO(ovr): Find what is going wrong with parallel tests & python on Linux -const darwinSuite = process.platform === 'darwin' && !native.isFallbackBuild() ? describe : xdescribe; const nativeInstance = new native.NativeInstance(); diff --git a/packages/cubejs-backend-native/tsconfig.json b/packages/cubejs-backend-native/tsconfig.json index fefa0570f6064..451dfb99dd029 100644 --- a/packages/cubejs-backend-native/tsconfig.json +++ b/packages/cubejs-backend-native/tsconfig.json @@ -2,7 +2,8 @@ "extends": "../../tsconfig.base.json", "include": [ "js", - "test" + "test", + "benchmarks" ], "exclude": [ "test/snapshotResolver.js" diff --git a/yarn.lock b/yarn.lock index 6687574ca525a..919c694a59df5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -10759,6 +10759,14 @@ before-after-hook@^2.2.0: resolved "https://registry.yarnpkg.com/before-after-hook/-/before-after-hook-2.2.2.tgz#a6e8ca41028d90ee2c24222f201c90956091613e" integrity sha512-3pZEU3NT5BFUo/AD5ERPWOgQOCZITni6iavr5AUw5AUwQjMlI0kzu5btnyD39AF0gUEsDPwJT+oY1ORBJijPjQ== +benchmark@^2.1.4: + version "2.1.4" + resolved "https://registry.yarnpkg.com/benchmark/-/benchmark-2.1.4.tgz#09f3de31c916425d498cc2ee565a0ebf3c2a5629" + integrity sha512-l9MlfN4M1K/H2fbhfMy3B7vJd6AGKJVQn2h6Sg/Yx+KckoUA7ewS5Vv6TjSq18ooE1kS9hhAlQRH3AkXIh/aOQ== + dependencies: + lodash "^4.17.4" + platform "^1.3.3" + best-effort-json-parser@^1.1.2: version "1.1.2" resolved "https://registry.yarnpkg.com/best-effort-json-parser/-/best-effort-json-parser-1.1.2.tgz#869272c9de76fc7d336c4d9e3a8bbcdee3edda04" @@ -16976,6 +16984,18 @@ javascript-stringify@^2.0.1: resolved "https://registry.yarnpkg.com/javascript-stringify/-/javascript-stringify-2.1.0.tgz#27c76539be14d8bd128219a2d731b09337904e79" integrity sha512-JVAfqNPTvNq3sB/VHQJAFxN/sPgKnsKrCwyRt15zwNCdrMMJDdcEOdubuy+DuJYYdm0ox1J4uzEuYKkN+9yhVg== +jest-bench@^29: + version "29.7.1" + resolved "https://registry.yarnpkg.com/jest-bench/-/jest-bench-29.7.1.tgz#2fd94552d9c8f4a6df8351785b067cc052ec2b19" + integrity sha512-eFjQa+KVThwqY+Ecs9jeD+CdTUlDrJUAAFLy+DlWW5H1crnG1F4ad5Dk8K+kV6nB2aGCdFcusKBdgtx1SXYiHQ== + dependencies: + "@jest/globals" "^29.7.0" + "@jest/reporters" "^29.7.0" + benchmark "^2.1.4" + chalk "^4.1.0" + lodash "^4.17.20" + ndjson "^2.0.0" + jest-changed-files@^27.5.1: version "27.5.1" resolved "https://registry.yarnpkg.com/jest-changed-files/-/jest-changed-files-27.5.1.tgz#a348aed00ec9bf671cc58a66fcbe7c3dfd6a68f5" @@ -19670,6 +19690,17 @@ ndjson@^1.3.0: split2 "^2.1.0" through2 "^2.0.3" +ndjson@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/ndjson/-/ndjson-2.0.0.tgz#320ac86f6fe53f5681897349b86ac6f43bfa3a19" + integrity sha512-nGl7LRGrzugTtaFcJMhLbpzJM6XdivmbkdlaGcrk/LXg2KL/YBC6z1g70xh0/al+oFuVFP8N8kiWRucmeEH/qQ== + dependencies: + json-stringify-safe "^5.0.1" + minimist "^1.2.5" + readable-stream "^3.6.0" + split2 "^3.0.0" + through2 "^4.0.0" + needle@^3.1.0: version "3.3.1" resolved "https://registry.yarnpkg.com/needle/-/needle-3.3.1.tgz#63f75aec580c2e77e209f3f324e2cdf3d29bd049" @@ -21099,6 +21130,11 @@ pkg-types@^1.0.3: mlly "^1.2.0" pathe "^1.1.0" +platform@^1.3.3: + version "1.3.6" + resolved "https://registry.yarnpkg.com/platform/-/platform-1.3.6.tgz#48b4ce983164b209c2d45a107adb31f473a6e7a7" + integrity sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg== + please-upgrade-node@^3.2.0: version "3.2.0" resolved "https://registry.yarnpkg.com/please-upgrade-node/-/please-upgrade-node-3.2.0.tgz#aeddd3f994c933e4ad98b99d9a556efa0e2fe942" @@ -22629,7 +22665,7 @@ readable-stream@2.3.7, readable-stream@^2.0.1, readable-stream@^2.0.5, readable- string_decoder "~1.1.1" util-deprecate "~1.0.1" -readable-stream@^3.0.0, readable-stream@^3.0.2, readable-stream@^3.0.6, readable-stream@^3.1.1, readable-stream@^3.4.0, readable-stream@^3.5.0, readable-stream@^3.6.0, readable-stream@^3.6.2: +readable-stream@3, readable-stream@^3.0.0, readable-stream@^3.0.2, readable-stream@^3.0.6, readable-stream@^3.1.1, readable-stream@^3.4.0, readable-stream@^3.5.0, readable-stream@^3.6.0, readable-stream@^3.6.2: version "3.6.2" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-3.6.2.tgz#56a9b36ea965c00c5a93ef31eb111a0f11056967" integrity sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA== @@ -23941,7 +23977,7 @@ split2@^2.1.0: dependencies: through2 "^2.0.2" -split2@^3.2.2: +split2@^3.0.0, split2@^3.2.2: version "3.2.2" resolved "https://registry.yarnpkg.com/split2/-/split2-3.2.2.tgz#bf2cf2a37d838312c249c89206fd7a17dd12365f" integrity sha512-9NThjpgZnifTkJpzTZ7Eue85S49QwpNhZTq6GRJwObb6jnLFNGB7Qm73V5HewTROPyxD0C29xqmaI68bQtV+hg== @@ -24846,6 +24882,13 @@ through2@^2.0.0, through2@^2.0.2, through2@^2.0.3: readable-stream "~2.3.6" xtend "~4.0.1" +through2@^4.0.0: + version "4.0.2" + resolved "https://registry.yarnpkg.com/through2/-/through2-4.0.2.tgz#a7ce3ac2a7a8b0b966c80e7c49f0484c3b239764" + integrity sha512-iOqSav00cVxEEICeD7TjLB1sueEL+81Wpzp2bY17uZjZN0pWZPuo4suZ/61VujxmqSGFfgOcNuTZ85QJwNZQpw== + dependencies: + readable-stream "3" + through@2, "through@>=2.2.7 <3", through@^2.3.4, through@^2.3.6, through@^2.3.8: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5"