diff --git a/css/chart.css b/css/chart.css index 1098c15..81c4c63 100644 --- a/css/chart.css +++ b/css/chart.css @@ -21,6 +21,11 @@ height: 250px; } +.chart-container.updating { + filter: blur(2px); + opacity: 0.7; +} + #chart { width: 100%; height: 100%; diff --git a/js/breakdowns/index.js b/js/breakdowns/index.js index c054267..5d6ddac 100644 --- a/js/breakdowns/index.js +++ b/js/breakdowns/index.js @@ -11,7 +11,7 @@ */ import { DATABASE } from '../config.js'; import { state } from '../state.js'; -import { query, getQueryErrorDetails, isAbortError } from '../api.js'; +import { getQueryErrorDetails, isAbortError } from '../api.js'; import { startRequestContext, getRequestContext, isRequestCurrent, mergeAbortSignals, } from '../request-context.js'; @@ -21,17 +21,16 @@ import { import { allBreakdowns as defaultBreakdowns } from './definitions.js'; import { renderBreakdownTable, renderBreakdownError, getNextTopN } from './render.js'; import { compileFilters } from '../filter-sql.js'; -import { getFiltersForColumn } from '../filters.js'; + import { loadSql } from '../sql-loader.js'; import { createLimiter } from '../concurrency-limiter.js'; import { fetchBreakdownData as fetchCoralogixBreakdown, } from '../coralogix/adapter.js'; -// Intentionally limits only breakdown queries: breakdowns fan out 20+ parallel -// queries (one per facet), the only code path with bulk parallelism. Chart, logs, -// and autocomplete each fire 1-2 queries and don't need limiting. -const queryLimiter = createLimiter(1); +// Soft cap on Coralogix breakdown concurrency — guards against unbounded +// parallelism if the breakdown list grows significantly. +const coralogixQueryLimiter = createLimiter(30); export function getBreakdowns() { return state.breakdowns?.length ? state.breakdowns : defaultBreakdowns; @@ -132,64 +131,6 @@ function fillExpectedLabels(data, b) { }); } -/** - * Fetch and append missing filtered values to data - */ -async function appendMissingFilteredValues(data, b, col, aggs, queryParams, requestStatus) { - const { isCurrent, signal } = requestStatus || {}; - const shouldApply = () => (typeof isCurrent === 'function' ? isCurrent() : true); - const { originalCol, isBytes, mult } = queryParams; - const filtersForCol = getFiltersForColumn(originalCol); - if (filtersForCol.length === 0 || b.getExpectedLabels) return data; - - const existingDims = new Set(data.map((row) => row.dim)); - const missingFilterValues = filtersForCol - .map((f) => f.value) - .filter((v) => v !== '' && !existingDims.has(v)); - - if (missingFilterValues.length === 0) return data; - - const searchCol = b.filterCol || col; - const valuesList = missingFilterValues - .map((v) => `'${v.replace(/'/g, "''")}'`) - .join(', '); - - const missingValuesSql = await loadSql('breakdown-missing', { - col, - aggTotal: isBytes ? `sum(\`response.headers.content_length\`)${mult}` : `count()${mult}`, - aggOk: aggs.aggOk, - agg4xx: aggs.agg4xx, - agg5xx: aggs.agg5xx, - database: DATABASE, - table: getTable(), - sampleClause: queryParams.sampleClause, - timeFilter: queryParams.timeFilter, - hostFilter: queryParams.hostFilter, - extra: queryParams.extra, - additionalWhereClause: state.additionalWhereClause, - searchCol, - valuesList, - }); - - try { - if (!shouldApply()) return data; - const missingResult = await query(missingValuesSql, { signal }); - if (!shouldApply()) return data; - if (missingResult.data && missingResult.data.length > 0) { - const markedRows = missingResult.data.map((row) => ({ - ...row, - isFilteredValue: true, - })); - return [...data, ...markedRows]; - } - } catch (err) { - if (!shouldApply()) return data; - if (isAbortError(err)) return data; - // Silently ignore errors fetching filtered values - } - return data; -} - /** * Build SQL query parameters for breakdown */ @@ -426,7 +367,7 @@ async function fetchBreakdownData(b, timeFilter, hostFilter, requestStatus) { // Get facet column name (handle function-based columns) const facetCol = typeof b.col === 'function' ? b.col(state.topN) : b.col; - // Build params object for compatibility with appendMissingFilteredValues + // Build params object for renderBreakdownTable compatibility const params = { col: facetCol, originalCol: facetCol, @@ -442,8 +383,8 @@ async function fetchBreakdownData(b, timeFilter, hostFilter, requestStatus) { const startTime = performance.now(); - // Call Coralogix adapter - const result = await queryLimiter(() => fetchCoralogixBreakdown({ + // Call Coralogix adapter (rate-limited to coralogixQueryLimiter slots) + const result = await coralogixQueryLimiter(() => fetchCoralogixBreakdown({ facet: facetCol, topN: state.topN, filters: state.filters, @@ -463,8 +404,6 @@ async function fetchBreakdownData(b, timeFilter, hostFilter, requestStatus) { const summaryRatio = getSummaryRatio(b, totals); const data = fillExpectedLabels(resultData, b); - // Note: appendMissingFilteredValues uses ClickHouse queries - skip for now with Coralogix - // data = await appendMissingFilteredValues(data, b, params.col, aggs, params, requestStatus); if (!isCurrent()) return null; return { @@ -601,148 +540,6 @@ export function increaseTopN(topNSelectEl, saveStateToURL, loadAllBreakdownsFn) } } -// --- Preview breakdowns during time range selection --- - -const HOUR_MS = 60 * 60 * 1000; - -function formatPreviewDateTime(date) { - return date.toISOString().replace('T', ' ').slice(0, 19); -} - -function getPreviewTimeFilter(start, end) { - const startIso = formatPreviewDateTime(start); - const endIso = formatPreviewDateTime(end); - return `toStartOfMinute(timestamp) BETWEEN toStartOfMinute(toDateTime('${startIso}')) AND toStartOfMinute(toDateTime('${endIso}'))`; -} - -function getPreviewSamplingConfig(durationMs) { - if (durationMs <= HOUR_MS) { - return { sampleClause: '', multiplier: 1 }; - } - const ratio = HOUR_MS / durationMs; - const sampleRate = Math.max(Math.round(ratio * 10000) / 10000, 0.0001); - const multiplier = Math.round(1 / sampleRate); - return { sampleClause: `SAMPLE ${sampleRate}`, multiplier }; -} - -function buildPreviewQueryParams(b, col, timeFilter, hostFilter, sampling) { - const originalCol = typeof b.col === 'function' ? b.col(state.topN) : b.col; - const hasActiveFilter = b.filterOp === 'LIKE' && b.filterCol - && state.filters.some((f) => f.col === originalCol); - const actualCol = hasActiveFilter ? b.filterCol : col; - - const mode = b.modeToggle ? state[b.modeToggle] : 'count'; - const isBytes = mode === 'bytes'; - const { sampleClause, multiplier } = sampling; - const mult = multiplier > 1 ? ` * ${multiplier}` : ''; - - return { - col: actualCol, - originalCol, - hasActiveFilter, - isBytes, - sampleClause, - mult, - extra: b.extraFilter || '', - facetFilters: getFacetFiltersExcluding(originalCol), - timeFilter, - hostFilter, - }; -} - -async function buildPreviewBreakdownSql(b, timeFilter, hostFilter, facetTimes, sampling) { - const baseCol = typeof b.col === 'function' ? b.col(state.topN) : b.col; - - if (canUseFacetTable(b)) { - const { startTime, endTime } = facetTimes; - const dimFilter = b.extraFilter ? "AND dim != ''" : ''; - const hasSummary = !!b.summaryDimCondition; - const sql = await loadSql('breakdown-facet', { - database: DATABASE, - facetName: b.facetName, - startTime, - endTime, - dimFilter, - innerSummaryCol: hasSummary - ? `,\n if(${b.summaryDimCondition}, cnt, 0) as summary_cnt` - : '', - summaryCol: hasSummary - ? ',\n sum(summary_cnt) as summary_cnt' - : '', - orderBy: b.orderBy || 'cnt DESC', - topN: String(state.topN), - }); - - const params = { - col: baseCol, - originalCol: baseCol, - hasActiveFilter: false, - isBytes: false, - sampleClause: '', - mult: '', - extra: '', - facetFilters: '', - timeFilter, - hostFilter, - }; - return { sql, params, aggs: buildAggregations(false, '') }; - } - - const params = buildPreviewQueryParams(b, baseCol, timeFilter, hostFilter, sampling); - const aggs = buildAggregations(params.isBytes, params.mult); - - if (b.rawCol && typeof b.col === 'function') { - const bucketExpr = b.col(state.topN, 'val'); - const innerSummary = b.summaryCountIf - ? `,\n countIf(${b.summaryCountIf})${params.mult} as summary_cnt` - : ''; - const outerSummary = b.summaryCountIf - ? ',\n sum(summary_cnt) as summary_cnt' - : ''; - - const sql = await loadSql('breakdown-bucketed', { - bucketExpr, - rawCol: b.rawCol, - ...aggs, - innerSummaryCol: innerSummary, - outerSummaryCol: outerSummary, - database: DATABASE, - table: getTable(), - sampleClause: params.sampleClause, - timeFilter, - hostFilter, - facetFilters: params.facetFilters, - extra: params.extra, - additionalWhereClause: state.additionalWhereClause, - topN: String(state.topN), - }); - - return { sql, params, aggs }; - } - - const summaryColWithMult = b.summaryCountIf - ? `,\n countIf(${b.summaryCountIf})${params.mult} as summary_cnt` - : ''; - - const sql = await loadSql('breakdown', { - col: params.col, - ...aggs, - summaryCol: summaryColWithMult, - database: DATABASE, - table: getTable(), - sampleClause: params.sampleClause, - timeFilter, - hostFilter, - facetFilters: params.facetFilters, - extra: params.extra, - additionalWhereClause: state.additionalWhereClause, - orderBy: b.orderBy || 'cnt DESC', - topN: String(state.topN), - }); - - return { sql, params, aggs }; -} - // Track whether preview is active for CSS indicator let previewActive = false; @@ -750,14 +547,7 @@ export function isPreviewActive() { return previewActive; } -async function loadPreviewBreakdown( - b, - timeFilter, - hostFilter, - facetTimes, - sampling, - requestStatus, -) { +async function loadPreviewBreakdown(b, start, end, hostFilter, requestStatus) { const { isCurrent, signal } = requestStatus; const card = document.getElementById(b.id); @@ -765,25 +555,31 @@ async function loadPreviewBreakdown( card.classList.add('updating'); + const facetCol = typeof b.col === 'function' ? b.col(state.topN) : b.col; + try { - const built = await buildPreviewBreakdownSql(b, timeFilter, hostFilter, facetTimes, sampling); - const { sql, params, aggs } = built; - const startTime = performance.now(); - const result = await queryLimiter(() => query(sql, { signal })); + const perfStart = performance.now(); + const result = await coralogixQueryLimiter(() => fetchCoralogixBreakdown({ + facet: facetCol, + topN: state.topN, + filters: state.filters, + hostFilter, + startTime: start, + endTime: end, + extraFilter: b.extraFilter || '', + signal, + })); if (!isCurrent()) return; - const elapsed = result.networkTime ?? (performance.now() - startTime); + const elapsed = result.networkTime ?? (performance.now() - perfStart); const summaryRatio = getSummaryRatio(b, result.totals); - - let data = fillExpectedLabels(result.data, b); - data = await appendMissingFilteredValues(data, b, params.col, aggs, params, requestStatus); - if (!isCurrent()) return; + const data = fillExpectedLabels(result.data, b); renderBreakdownTable( b.id, data, result.totals, - params.col, + facetCol, b.linkPrefix, b.linkSuffix, b.linkFn, @@ -795,9 +591,9 @@ async function loadPreviewBreakdown( b.summaryColor, b.modeToggle, !!b.getExpectedLabels, - params.hasActiveFilter ? null : b.filterCol, - params.hasActiveFilter ? null : b.filterValueFn, - params.hasActiveFilter ? null : b.filterOp, + b.filterCol, + b.filterValueFn, + b.filterOp, ); card.classList.add('preview'); @@ -821,24 +617,14 @@ export async function loadPreviewBreakdowns(selectionStart, selectionEnd) { signal: requestContext.signal, }; - const durationMs = selectionEnd - selectionStart; const start = new Date(Math.floor(selectionStart.getTime() / 60000) * 60000); const end = new Date(Math.ceil(selectionEnd.getTime() / 60000) * 60000); - - const timeFilter = getPreviewTimeFilter(start, end); const hostFilter = getHostFilter(); - const facetTimes = { - startTime: formatPreviewDateTime(start), - endTime: formatPreviewDateTime(end), - }; - const sampling = getPreviewSamplingConfig(durationMs); previewActive = true; const breakdowns = getBreakdowns(); await Promise.all( - breakdowns.map( - (b) => loadPreviewBreakdown(b, timeFilter, hostFilter, facetTimes, sampling, requestStatus), - ), + breakdowns.map((b) => loadPreviewBreakdown(b, start, end, hostFilter, requestStatus)), ); } diff --git a/js/chart.js b/js/chart.js index cf1887c..054bc30 100644 --- a/js/chart.js +++ b/js/chart.js @@ -18,7 +18,7 @@ import { isAbortError } from './api.js'; import { fetchTimeSeriesData } from './coralogix/adapter.js'; import { - getFacetFilters, loadPreviewBreakdowns, revertPreviewBreakdowns, isPreviewActive, + loadPreviewBreakdowns, revertPreviewBreakdowns, isPreviewActive, } from './breakdowns/index.js'; import { formatNumber } from './format.js'; import { getRequestContext, isRequestCurrent } from './request-context.js'; @@ -946,9 +946,11 @@ export async function loadTimeSeries(requestContext = getRequestContext('dashboa const { requestId, signal, scope } = requestContext; const isCurrent = () => isRequestCurrent(requestId, scope); const hostFilter = getHostFilter(); - const facetFilters = getFacetFilters(); + const facetFilters = state.filters; const bucket = getTimeBucket(); + const container = document.querySelector('.chart-container'); + container?.classList.add('updating'); try { // Fetch time series data using Coralogix adapter const data = await fetchTimeSeriesData({ @@ -968,5 +970,7 @@ export async function loadTimeSeries(requestContext = getRequestContext('dashboa if (!isCurrent() || isAbortError(err)) return; // eslint-disable-next-line no-console console.error('Chart error:', err); + } finally { + if (isCurrent()) container?.classList.remove('updating'); } } diff --git a/js/coralogix/adapter.js b/js/coralogix/adapter.js index ca4ff07..40f8aed 100644 --- a/js/coralogix/adapter.js +++ b/js/coralogix/adapter.js @@ -18,18 +18,25 @@ * transforms responses to match expected klickhaus data structures. */ -import { CORALOGIX_CONFIG, QUERY_TIERS } from './config.js'; +import { CORALOGIX_CONFIG } from './config.js'; import { getToken, getTeamId } from './auth.js'; import { authenticatedFetch } from './interceptor.js'; import { translateFacetFilters, translateHostFilter, getFieldPath, + translateColExpression, } from './filter-translator.js'; import { QueryError } from '../api.js'; import { TIME_RANGES } from '../constants.js'; +import { queryTimestamp, customTimeRange } from '../time.js'; import { parseNDJSON } from './ndjson-parser.js'; +export const EXECUTION_PROFILES = { + ACCURACY: 'EXECUTION_PROFILE_PRESET_ACCURACY', + PERFORMANCE: 'EXECUTION_PROFILE_PRESET_PERFORMANCE', +}; + // --------------------------------------------------------------------------- // Error parsing helpers (extracted to reduce parseCoralogixError complexity) // --------------------------------------------------------------------------- @@ -107,15 +114,20 @@ function mapClickHouseIntervalToDataPrime(clickhouseBucket) { return INTERVAL_MAP[clickhouseBucket] || '1m'; } -/** Compute start/end Date objects and tier from a time-range key. */ +/** Compute start/end Date objects from a time-range key. + * Mirrors getSelectedRange() in time.js: uses the frozen queryTimestamp when + * inside a dashboard load so all parallel queries share identical time bounds. + */ function resolveTimeRange(timeRange) { + const custom = customTimeRange(); + if (custom) { + return { startTime: new Date(custom.start), endTime: new Date(custom.end) }; + } const def = TIME_RANGES[timeRange]; if (!def) throw new Error(`Unknown time range: ${timeRange}`); - const endTime = new Date(); + const endTime = queryTimestamp() || new Date(); const startTime = new Date(endTime.getTime() - def.periodMs); - const hours = def.periodMs / (60 * 60 * 1000); - const tier = CORALOGIX_CONFIG.getTierForTimeRange(hours); - return { startTime, endTime, tier }; + return { startTime, endTime }; } // --------------------------------------------------------------------------- @@ -194,83 +206,6 @@ function transformBreakdownResult(result, facet = '') { return { data, totals }; } -// --------------------------------------------------------------------------- -// multiIf conversion helpers (extracted to reduce complexity) -// --------------------------------------------------------------------------- - -/** Split a string by commas, respecting quoted substrings. */ -function splitRespectingQuotes(str) { - const parts = []; - let current = ''; - let inQuote = false; - let quoteChar = ''; - for (let i = 0; i < str.length; i += 1) { - const ch = str[i]; - if ((ch === "'" || ch === '"') && (i === 0 || str[i - 1] !== '\\')) { - if (!inQuote) { - inQuote = true; - quoteChar = ch; - } else if (ch === quoteChar) { - inQuote = false; - } - } - if (ch === ',' && !inQuote) { - parts.push(current.trim()); - current = ''; - } else { - current += ch; - } - } - if (current) parts.push(current.trim()); - return parts; -} - -/** Parse condition/label pairs from the parts of a multiIf expression. */ -function parseMultiIfConditions(parts) { - const conditions = []; - let i = 0; - while (i < parts.length - 1) { - const condition = parts[i]; - const label = parts[i + 1].replace(/^['"]|['"]$/g, ''); - const ltMatch = condition.match(/`[^`]+`\s*<\s*(\d+)/); - const eqMatch = condition.match(/`[^`]+`\s*=\s*(\d+)/); - if (ltMatch) { - conditions.push({ op: '<', threshold: ltMatch[1], label }); - i += 2; - } else if (eqMatch) { - conditions.push({ op: '==', threshold: eqMatch[1], label }); - i += 2; - } else { break; } - } - return conditions; -} - -/** - * Convert ClickHouse multiIf() to Data Prime case_lessthan or case. - * @param {string} multiIfExpr - multiIf expression - * @returns {string} Data Prime case expression - */ -function convertMultiIfToCaseLessThan(multiIfExpr) { - const fieldMatch = multiIfExpr.match(/multiIf\s*\(\s*`([^`]+)`/i); - if (!fieldMatch) { - throw new Error(`Cannot extract field from multiIf: ${multiIfExpr}`); - } - const dpField = getFieldPath(`\`${fieldMatch[1]}\``); - const inner = multiIfExpr.replace(/^multiIf\s*\(/i, '').replace(/\)$/, ''); - const parts = splitRespectingQuotes(inner); - const conditions = parseMultiIfConditions(parts); - const fallback = parts[parts.length - 1].replace(/^['"]|['"]$/g, ''); - - if (conditions.some((c) => c.op === '==')) { - const cases = conditions.map((c) => (c.op === '==' - ? `${dpField}:num == ${c.threshold} -> '${c.label}'` - : `${dpField}:num < ${c.threshold} -> '${c.label}'`)); - return `case { ${cases.join(', ')}, _ -> '${fallback}' }`; - } - const list = conditions.map((c) => `${c.threshold} -> '${c.label}'`); - return `case_lessthan { ${dpField}:num, ${list.join(', ')}, _ -> '${fallback}' }`; -} - // --------------------------------------------------------------------------- // Filter / expression translation // --------------------------------------------------------------------------- @@ -292,43 +227,11 @@ function translateExtraFilter(extraFilter) { return filter; } -/** Build a Data Prime expression for facet grouping. */ -function buildFacetExpression(facetExpression) { - const cleanExpr = facetExpression.replace(/`/g, ''); - - if (cleanExpr === 'client.asn') { - return "concat($d.cdn.originating_ip_geoip.asn.number, ' - '," - + ' $d.cdn.originating_ip_geoip.asn.organization)'; - } - if (cleanExpr.includes('intDiv') && cleanExpr.includes('response.status')) { - return '$d.response.status:num / 100'; - } - if (cleanExpr.match(/^toString\(/)) { - return `${getFieldPath(facetExpression)}:string`; - } - if (cleanExpr.match(/^upper\(/)) return getFieldPath(facetExpression); - if (cleanExpr.match(/^REGEXP_REPLACE\(/i)) { - return getFieldPath(facetExpression); - } - if (cleanExpr.match(/^if\(/i)) { - if (cleanExpr.includes('x_forwarded_for')) { - return '$d.request.headers.x_forwarded_for'; - } - const m = cleanExpr.match(/if\([^,]+,\s*([^,]+),/i); - if (m) return getFieldPath(`\`${m[1].replace(/`/g, '').trim()}\``); - } - if (cleanExpr.match(/^multiIf\(/i)) { - return convertMultiIfToCaseLessThan(facetExpression); - } - return getFieldPath(facetExpression); -} - // --------------------------------------------------------------------------- // Shared aggregation snippet // --------------------------------------------------------------------------- -const AGG_COUNTS = `count() as cnt, - sum(case { $d.response.status:num < 400 -> 1, _ -> 0 }) as cnt_ok, +const AGG_STATUS = `sum(case { $d.response.status:num < 400 -> 1, _ -> 0 }) as cnt_ok, sum(case { $d.response.status:num >= 400 && $d.response.status:num < 500 -> 1, _ -> 0 }) as cnt_4xx, sum(case { $d.response.status:num >= 500 -> 1, _ -> 0 }) as cnt_5xx`; @@ -374,9 +277,9 @@ function buildTimeSeriesQuery({ function buildBreakdownQuery({ facet, topN, filters = [], hostFilter = '', - startTime, endTime, extraFilter = '', orderBy = 'cnt DESC', + startTime, endTime, extraFilter = '', }) { - const facetExpr = buildFacetExpression(facet); + const facetExpr = translateColExpression(facet); let query = `source logs between @'${startTime.toISOString()}' and @'${endTime.toISOString()}'`; // Exclude current facet from filters @@ -385,10 +288,7 @@ function buildBreakdownQuery({ hostFilter, filters: otherFilters, extraFilter, }); - query += `\n| groupby ${facetExpr} as dim aggregate\n ${AGG_COUNTS}`; - const orderField = orderBy.includes('DESC') ? orderBy.split(' ')[0] : 'cnt'; - const dir = orderBy.includes('ASC') ? 'asc' : 'desc'; - query += `\n| orderby ${orderField} ${dir}\n| limit ${topN}`; + query += `\n| top ${topN} ${facetExpr} as dim,\n ${AGG_STATUS}\n by count() as cnt`; return query; } @@ -415,11 +315,11 @@ function buildMultiFacetQuery({ query = appendFilters(query, { hostFilter, filters }); const sets = facets.map((def) => { - const expr = buildFacetExpression(def.col); + const expr = translateColExpression(def.col); const ob = def.orderBy || 'cnt DESC'; const field = ob.includes('DESC') ? ob.split(' ')[0] : 'cnt'; const dir = ob.includes('ASC') ? 'asc' : 'desc'; - return `(groupby ${expr} as dim aggregate\n ${AGG_COUNTS} + return `(groupby ${expr} as dim aggregate\n count() as cnt,\n ${AGG_STATUS} | create facet_id = '${def.id}' | orderby ${field} ${dir} | limit ${topN})`; @@ -462,7 +362,9 @@ async function sendDataPrimeRequest(url, headers, body, signal) { } /** Execute a Data Prime query against Coralogix API with retry for 429. */ -export async function executeDataPrimeQuery(dataPrimeQuery, { signal, tier } = {}) { +export async function executeDataPrimeQuery(dataPrimeQuery, { + signal, tier, executionProfile, +} = {}) { const token = getToken(); if (!token) { throw new QueryError('Coralogix authentication required', { @@ -470,13 +372,17 @@ export async function executeDataPrimeQuery(dataPrimeQuery, { signal, tier } = { }); } - const requestBody = JSON.stringify({ + const body = { query: dataPrimeQuery, metadata: { tier: tier || CORALOGIX_CONFIG.defaultTier, syntax: 'QUERY_SYNTAX_DATAPRIME', }, - }); + }; + if (executionProfile) { + body.executionProfile = { preset: executionProfile }; + } + const requestBody = JSON.stringify(body); const fetchStart = performance.now(); const teamId = getTeamId(); @@ -518,35 +424,34 @@ export async function executeDataPrimeQuery(dataPrimeQuery, { signal, tier } = { export async function fetchTimeSeriesData({ timeRange, interval, filters = [], hostFilter = '', signal, }) { - const { startTime, endTime, tier } = resolveTimeRange(timeRange); + const { startTime, endTime } = resolveTimeRange(timeRange); const dpInterval = mapClickHouseIntervalToDataPrime(interval); const query = buildTimeSeriesQuery({ startTime, endTime, interval: dpInterval, filters, hostFilter, }); - const result = await executeDataPrimeQuery(query, { signal, tier }); + const result = await executeDataPrimeQuery(query, { + signal, executionProfile: EXECUTION_PROFILES.PERFORMANCE, + }); return transformTimeSeriesResult(result); } -/** Fetch breakdown/facet data for tables. */ +/** Fetch breakdown/facet data for tables. + * Pass explicit startTime/endTime Date objects (e.g. for zoom previews) to + * bypass resolveTimeRange; otherwise timeRange key is used as normal. + */ export async function fetchBreakdownData({ facet, topN, filters = [], hostFilter = '', - timeRange, extraFilter = '', orderBy = 'cnt DESC', signal, + timeRange, startTime: explicitStart, endTime: explicitEnd, + extraFilter = '', orderBy = 'cnt DESC', signal, }) { - const { startTime, endTime } = resolveTimeRange(timeRange); + const { startTime, endTime } = (explicitStart && explicitEnd) + ? { startTime: explicitStart, endTime: explicitEnd } + : resolveTimeRange(timeRange); const query = buildBreakdownQuery({ - facet, - topN, - filters, - hostFilter, - startTime, - endTime, - extraFilter, - orderBy, + facet, topN, filters, hostFilter, startTime, endTime, extraFilter, orderBy, }); - // Always use FREQUENT_SEARCH for breakdowns -- ARCHIVE fails on - // high-cardinality groupby queries over long time ranges. const result = await executeDataPrimeQuery(query, { - signal, tier: QUERY_TIERS.FREQUENT_SEARCH, + signal, executionProfile: EXECUTION_PROFILES.PERFORMANCE, }); const transformed = transformBreakdownResult(result, facet); transformed.networkTime = result.networkTime; @@ -557,11 +462,11 @@ export async function fetchBreakdownData({ export async function fetchAllBreakdowns({ facets, timeRange, filters = [], hostFilter = '', topN, signal, }) { - const { startTime, endTime, tier } = resolveTimeRange(timeRange); + const { startTime, endTime } = resolveTimeRange(timeRange); const query = buildMultiFacetQuery({ facets, startTime, endTime, filters, hostFilter, topN, }); - const result = await executeDataPrimeQuery(query, { signal, tier }); + const result = await executeDataPrimeQuery(query, { signal }); const byId = {}; for (const row of result.results || []) { @@ -582,11 +487,11 @@ export async function fetchAllBreakdowns({ export async function fetchLogsData({ filters = [], hostFilter = '', timeRange, limit, offset = 0, signal, }) { - const { startTime, endTime, tier } = resolveTimeRange(timeRange); + const { startTime, endTime } = resolveTimeRange(timeRange); const query = buildLogsQuery({ filters, hostFilter, startTime, endTime, limit, offset, }); - const result = await executeDataPrimeQuery(query, { signal, tier }); + const result = await executeDataPrimeQuery(query, { signal }); return transformLogsResult(result); } diff --git a/js/coralogix/config.js b/js/coralogix/config.js index e9dca83..2099748 100644 --- a/js/coralogix/config.js +++ b/js/coralogix/config.js @@ -51,7 +51,7 @@ function getEnv(key) { if (typeof window !== 'undefined' && !window.ENV) { window.ENV = { CX_TEAM_ID: '7667', - CX_DATAPRIME_URL: 'https://api.coralogix.com/api/v1/dataprime/query', + CX_DATAPRIME_URL: 'https://ng-api-http.coralogix.com/api/v1/dataprime/query', CX_GRPC_GATEWAY_URL: 'https://api.coralogix.com', CX_HTTP_GATEWAY_URL: 'https://api.coralogix.com', CX_BASE_URL: 'https://api.coralogix.com', @@ -67,7 +67,7 @@ if (typeof window !== 'undefined' && !window.ENV) { */ export const CORALOGIX_CONFIG = { // API Endpoints - dataprimeApiUrl: getEnv('CX_DATAPRIME_URL') || 'https://api.coralogix.com/api/v1/dataprime/query', + dataprimeApiUrl: getEnv('CX_DATAPRIME_URL') || 'https://ng-api-http.coralogix.com/api/v1/dataprime/query', grpcGatewayUrl: getEnv('CX_GRPC_GATEWAY_URL') || 'https://ng-api-grpc.coralogix.com', httpGatewayUrl: getEnv('CX_HTTP_GATEWAY_URL') || 'https://ng-api-http.coralogix.com', baseApiUrl: getEnv('CX_BASE_URL') || 'https://api.coralogix.com', diff --git a/js/coralogix/filter-translator.js b/js/coralogix/filter-translator.js index e2dd057..0819239 100644 --- a/js/coralogix/filter-translator.js +++ b/js/coralogix/filter-translator.js @@ -217,9 +217,13 @@ export function translateFilter(filter) { const value = filter.filterValue ?? filter.value; const operator = filter.filterOp || '='; - // Get Data Prime field path - const fieldPath = getFieldPath(column); - const escapedValue = escapeValue(value); + // Use the full expression translator so filter expressions always match the + // groupby expression produced by translateColExpression (e.g. intDiv → floor). + // eslint-disable-next-line no-use-before-define + const fieldPath = translateColExpression(column); + // Empty string means "no value" — compare against null in Data Prime rather + // than '' so it works correctly for any expression, including complex ones. + const escapedValue = value === '' ? 'null' : escapeValue(value); // Handle different operators if (operator === 'LIKE') { @@ -416,3 +420,118 @@ export function translateInFilter(column, values, exclude = false) { const escapedValues = values.map((v) => escapeValue(v)).join(', '); return `${fieldPath}.in([${escapedValues}])`; } + +// --------------------------------------------------------------------------- +// Full ClickHouse expression → Data Prime expression translation +// --------------------------------------------------------------------------- + +/** Split a string by commas, respecting quoted substrings. */ +function splitRespectingQuotes(str) { + const parts = []; + let current = ''; + let inQuote = false; + let quoteChar = ''; + for (let i = 0; i < str.length; i += 1) { + const ch = str[i]; + if ((ch === "'" || ch === '"') && (i === 0 || str[i - 1] !== '\\')) { + if (!inQuote) { + inQuote = true; + quoteChar = ch; + } else if (ch === quoteChar) { + inQuote = false; + } + } + if (ch === ',' && !inQuote) { + parts.push(current.trim()); + current = ''; + } else { + current += ch; + } + } + if (current) parts.push(current.trim()); + return parts; +} + +/** Parse condition/label pairs from the parts of a multiIf expression. */ +function parseMultiIfConditions(parts) { + const conditions = []; + let i = 0; + while (i < parts.length - 1) { + const condition = parts[i]; + const label = parts[i + 1].replace(/^['"]|['"]$/g, ''); + const ltMatch = condition.match(/`[^`]+`\s*<\s*(\d+)/); + const eqMatch = condition.match(/`[^`]+`\s*=\s*(\d+)/); + if (ltMatch) { + conditions.push({ op: '<', threshold: ltMatch[1], label }); + i += 2; + } else if (eqMatch) { + conditions.push({ op: '==', threshold: eqMatch[1], label }); + i += 2; + } else { break; } + } + return conditions; +} + +/** Convert ClickHouse multiIf() to Data Prime case_lessthan or case. */ +function convertMultiIfToCaseLessThan(multiIfExpr) { + const fieldMatch = multiIfExpr.match(/multiIf\s*\(\s*`([^`]+)`/i); + if (!fieldMatch) { + throw new Error(`Cannot extract field from multiIf: ${multiIfExpr}`); + } + const dpField = getFieldPath(`\`${fieldMatch[1]}\``); + const inner = multiIfExpr.replace(/^multiIf\s*\(/i, '').replace(/\)$/, ''); + const parts = splitRespectingQuotes(inner); + const conditions = parseMultiIfConditions(parts); + const fallback = parts[parts.length - 1].replace(/^['"]|['"]$/g, ''); + + if (conditions.some((c) => c.op === '==')) { + const cases = conditions.map((c) => (c.op === '==' + ? `${dpField}:num == ${c.threshold} -> '${c.label}'` + : `${dpField}:num < ${c.threshold} -> '${c.label}'`)); + return `case { ${cases.join(', ')}, _ -> '${fallback}' }`; + } + const list = conditions.map((c) => `${c.threshold} -> '${c.label}'`); + return `case_lessthan { ${dpField}:num, ${list.join(', ')}, _ -> '${fallback}' }`; +} + +/** + * Translate a full ClickHouse column/expression to its Data Prime equivalent. + * This is the single source of truth used by both the groupby (top) clause and + * filter expressions, guaranteeing they always produce the same string so that + * filter comparisons are valid. + * + * @param {string} col - ClickHouse expression (e.g. col from breakdown definition) + * @returns {string} Data Prime expression + */ +export function translateColExpression(col) { + const cleanExpr = col.replace(/`/g, ''); + + if (cleanExpr === 'client.asn') { + return "concat($d.cdn.originating_ip_geoip.asn.number, ' - '," + + ' $d.cdn.originating_ip_geoip.asn.organization)'; + } + if (cleanExpr.includes('intDiv') && cleanExpr.includes('response.status')) { + return 'if($d.response.status != null, `{floor($d.response.status:num/100)}xx`, null)'; + } + if (cleanExpr.match(/^toString\(/)) { + return `${getFieldPath(col)}:string`; + } + const castStringMatch = cleanExpr.match(/^CAST\((.+),\s*'String'\)$/i); + if (castStringMatch) { + const inner = `\`${castStringMatch[1].trim().replace(/`/g, '')}\``; + return `(${getFieldPath(inner)}):string`; + } + if (cleanExpr.match(/^upper\(/)) return getFieldPath(col); + if (cleanExpr.match(/^REGEXP_REPLACE\(/i)) return getFieldPath(col); + if (cleanExpr.match(/^if\(/i)) { + if (cleanExpr.includes('x_forwarded_for')) { + return '$d.request.headers.x_forwarded_for'; + } + const m = cleanExpr.match(/if\([^,]+,\s*([^,]+),/i); + if (m) return getFieldPath(`\`${m[1].replace(/`/g, '').trim()}\``); + } + if (cleanExpr.match(/^multiIf\(/i)) { + return convertMultiIfToCaseLessThan(col); + } + return getFieldPath(col); +} diff --git a/js/coralogix/filter-translator.test.js b/js/coralogix/filter-translator.test.js index 89d8990..df8f739 100644 --- a/js/coralogix/filter-translator.test.js +++ b/js/coralogix/filter-translator.test.js @@ -183,7 +183,7 @@ describe('filter-translator', () => { value: '', exclude: false, }; - assert.strictEqual(translateFilter(filter), "$d.request.headers.referer == ''"); + assert.strictEqual(translateFilter(filter), '$d.request.headers.referer == null'); }); it('should escape quotes in values', () => {