Skip to content

Commit 39c0a60

Browse files
omkarK06omkarK06
committed
fix: fixed logs full range volume
1 parent 6e04d6c commit 39c0a60

File tree

6 files changed

+152
-85
lines changed

6 files changed

+152
-85
lines changed

‎src/.DS_Store‎

0 Bytes
Binary file not shown.

‎src/components/QueryEditor.tsx‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,6 @@ export const QueryEditor = ({ query, onChange, onRunQuery, datasource, app, data
2020
const [displayOptions, setDisplayOptions]: any = useState([]);
2121

2222

23-
const isInDashboard = useMemo(() => app === 'panel-editor', [app]);
24-
2523
const getTimeStampColumnName = () => {
2624
return datasource.instanceSettings?.jsonData?.timestamp_column || '_timestamp';
2725
};
@@ -41,6 +39,8 @@ export const QueryEditor = ({ query, onChange, onRunQuery, datasource, app, data
4139
{ label: 'Force Logs', value: 'logs' },
4240
]);
4341
}, []);
42+
43+
const isInDashboard = useMemo(() => app === 'panel-editor', [app]);
4444

4545
useEffect(() => {
4646
startLoading();

‎src/datasource.ts‎

Lines changed: 139 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ export class DataSource
3030
streamFields: any[];
3131
cachedQuery: CachedQuery;
3232
timestampColumn: string;
33+
histogramQuery: any;
3334

3435
constructor(instanceSettings: DataSourceInstanceSettings<MyDataSourceOptions>) {
3536
super(instanceSettings);
@@ -43,6 +44,7 @@ export class DataSource
4344
promise: null,
4445
};
4546
this.timestampColumn = instanceSettings.jsonData.timestamp_column;
47+
this.histogramQuery = null;
4648
}
4749

4850
applyTemplateVariables(query: MyQuery, scopedVars: any): MyQuery {
@@ -54,9 +56,14 @@ export class DataSource
5456

5557
async query(options: DataQueryRequest<MyQuery>): Promise<DataQueryResponse> {
5658
const timestamps = getConsumableTime(options.range);
59+
// console.log('timestamps', timestamps);
60+
// console.log('options', options);
5761
const interpolatedTargets = options.targets.map((target) => {
5862
return this.applyTemplateVariables(target, options.scopedVars);
5963
});
64+
65+
// console.log("options", options);
66+
6067
const promises = interpolatedTargets.map((target) => {
6168
if (!this.cachedQuery.data) {
6269
this.cachedQuery.data = new Promise((resolve, reject) => {
@@ -66,21 +73,40 @@ export class DataSource
6673
};
6774
});
6875
}
69-
const reqData = buildQuery(target, timestamps, this.streamFields, options.app, this.timestampColumn);
76+
77+
let reqData = buildQuery(target, timestamps, this.streamFields, options.app, this.timestampColumn);
78+
79+
// // If its a logs search query, we need to save the histogram query from the request
80+
// // We store it in the histogramQuery variable and use it later for the graph query if the request is for a graph
81+
if (!target.refId?.includes(REF_ID_STARTER_LOG_VOLUME)) {
82+
this.histogramQuery = reqData;
83+
} else if (target.refId?.includes(REF_ID_STARTER_LOG_VOLUME) && this.histogramQuery) {
84+
reqData = this.histogramQuery;
85+
reqData.query.sql_mode = 'context';
86+
delete reqData.query.size;
87+
}
88+
7089
const cacheKey = JSON.stringify({
7190
reqData,
7291
displayMode: target.displayMode ?? 'auto',
92+
type: target.refId,
7393
});
74-
94+
95+
console.log('cacheKey', cacheKey);
96+
console.log('cached request query', this.cachedQuery);
97+
console.log('Is same', this.cachedQuery.requestQuery === cacheKey);
7598
if (cacheKey === this.cachedQuery.requestQuery) {
7699
return this.cachedQuery.data
77100
?.then((res) => {
101+
console.log('res in cache', target, res);
78102
const mode = target.displayMode || 'auto';
103+
// console.log('mode', mode);
104+
// console.log('res in cache', res);
79105
if (target?.refId?.includes(REF_ID_STARTER_LOG_VOLUME)) {
80106
return res.graph;
81107
}
82108
if (options.app === 'panel-editor' || options.app === 'dashboard') {
83-
if(mode === 'graph' || mode === 'auto') {
109+
if (mode === 'graph' || mode === 'auto') {
84110
return res.graph;
85111
}
86112
}
@@ -100,24 +126,85 @@ export class DataSource
100126
this.cachedQuery.isFetching = true;
101127
return this.doRequest(target, reqData)
102128
.then((response) => {
103-
if (options.app === 'panel-editor' || options.app === 'dashboard') {
104-
const mode = target.displayMode || 'auto';
105-
const logsDf = getLogsDataFrame(response.hits, target, this.streamFields, this.timestampColumn);
106-
const graphDf = getGraphDataFrame(response.hits, target, options.app, this.timestampColumn);
107-
this.cachedQuery.promise?.resolve({ graph: graphDf, logs: logsDf });
108-
return mode === 'logs' ? logsDf : graphDf;
109-
}
129+
// if (options.app === 'panel-editor' || options.app === 'dashboard') {
130+
// const mode = target.displayMode || 'auto';
131+
// const logsDf = getLogsDataFrame(response.hits, target, this.streamFields, this.timestampColumn);
132+
// const graphDf = getGraphDataFrame(response.hits, target, options.app, this.timestampColumn);
133+
// this.cachedQuery.promise?.resolve({ graph: graphDf, logs: logsDf });
134+
// return mode === 'logs' ? logsDf : graphDf;
135+
// }
136+
137+
// Handle histogram queries for log volume using partitions
138+
if (target?.refId?.includes(REF_ID_STARTER_LOG_VOLUME) && this.histogramQuery) {
139+
// First, get partition information for histogram queries
140+
return this.doPartitionRequest(target, this.histogramQuery.query)
141+
.then((partitionResponse) => {
142+
// Check if partitions are available
143+
if (partitionResponse?.partitions?.length > 0) {
144+
// Use partitions to make histogram requests
145+
const partitions = partitionResponse.partitions;
146+
147+
if (!partitionResponse.is_histogram_eligible) {
148+
return null;
149+
}
110150

111-
const logsDataFrame = getLogsDataFrame(response.hits, target, this.streamFields, this.timestampColumn);
151+
const histogramPromises = partitions.map((partition: any) => {
152+
// Create histogram query for each partition
153+
const partitionHistogramQuery = {
154+
...this.histogramQuery,
155+
query: {
156+
...this.histogramQuery.query,
157+
start_time: partition[0],
158+
end_time: partition[1],
159+
histogram_interval: partitionResponse.histogram_interval,
160+
},
161+
};
112162

113-
const graphDataFrame = getGraphDataFrame(response?.aggs?.histogram || [], target, options.app, this.timestampColumn);
163+
return this.doHistogramRequest(target, partitionHistogramQuery);
164+
});
114165

115-
this.cachedQuery.promise?.resolve({ graph: graphDataFrame, logs: logsDataFrame });
166+
// Combine results from all partitions
167+
return Promise.all(histogramPromises).then((histogramResponses) => {
168+
// Merge histogram data from all partitions
169+
const combinedHits = histogramResponses.reduce((acc, response) => {
170+
return acc.concat(response.hits || []);
171+
}, []);
116172

117-
if (target?.refId?.includes(REF_ID_STARTER_LOG_VOLUME)) {
173+
const graphDataFrame = getGraphDataFrame(combinedHits, target, options.app, 'zo_sql_key');
174+
this.cachedQuery.promise?.resolve({ graph: graphDataFrame, logs: null });
175+
176+
return graphDataFrame;
177+
});
178+
} else {
179+
// Fallback to direct histogram request if no partitions
180+
return this.doHistogramRequest(target, this.histogramQuery).then((histogramResponse) => {
181+
const graphDataFrame = getGraphDataFrame(
182+
histogramResponse.hits,
183+
target,
184+
options.app,
185+
this.timestampColumn
186+
);
187+
this.cachedQuery.promise?.resolve({ graph: graphDataFrame, logs: null });
188+
return graphDataFrame;
189+
});
190+
}
191+
})
192+
.catch((error) => {
193+
console.error('Partition or histogram request failed:', error);
194+
// Fallback to empty graph
195+
const graphDataFrame = getGraphDataFrame([], target, options.app, this.timestampColumn);
196+
this.cachedQuery.promise?.resolve({ graph: graphDataFrame, logs: null });
197+
return graphDataFrame;
198+
});
199+
} else if (target?.refId?.includes(REF_ID_STARTER_LOG_VOLUME)) {
200+
const graphDataFrame = getGraphDataFrame([], target, options.app, this.timestampColumn);
201+
this.cachedQuery.promise?.resolve({ graph: graphDataFrame, logs: null });
118202
return graphDataFrame;
203+
} else {
204+
const logsDataFrame = getLogsDataFrame(response.hits, target, this.streamFields, this.timestampColumn);
205+
this.cachedQuery.promise?.resolve({ graph: null, logs: logsDataFrame });
206+
return logsDataFrame;
119207
}
120-
return logsDataFrame;
121208
})
122209
.catch((err) => {
123210
this.cachedQuery.promise?.reject(err);
@@ -149,7 +236,41 @@ export class DataSource
149236
}
150237

151238
doRequest(target: any, data: any) {
152-
return getBackendSrv().post(this.url + `/api/${target.organization}/_search?type=logs`, data, {
239+
const searchType = 'ui';
240+
const useCache = true;
241+
const pageType = 'logs';
242+
243+
const url =
244+
this.url + `/api/${target.organization}/_search?type=${pageType}&search_type=${searchType}&use_cache=${useCache}`;
245+
246+
return getBackendSrv().post(url, data, {
247+
showErrorAlert: false,
248+
});
249+
}
250+
251+
doPartitionRequest(target: any, data: any) {
252+
const pageType = 'logs';
253+
const enableAlignHistogram = true;
254+
255+
const url =
256+
this.url +
257+
`/api/${target.organization}/_search_partition?type=${pageType}&enable_align_histogram=${enableAlignHistogram}`;
258+
259+
return getBackendSrv().post(url, data, {
260+
showErrorAlert: false,
261+
});
262+
}
263+
264+
doHistogramRequest(target: any, data: any) {
265+
const searchType = 'ui';
266+
const useCache = true;
267+
const pageType = 'logs';
268+
269+
const url =
270+
this.url +
271+
`/api/${target.organization}/_search?type=${pageType}&search_type=${searchType}&use_cache=${useCache}&is_ui_histogram=true`;
272+
273+
return getBackendSrv().post(url, data, {
153274
showErrorAlert: false,
154275
});
155276
}
@@ -217,6 +338,8 @@ export class DataSource
217338
if (!this.getSupportedSupplementaryQueryTypes().includes(type)) {
218339
return undefined;
219340
}
341+
// console.log('type', type);
342+
// console.log('request', request);
220343

221344
switch (type) {
222345
case SupplementaryQueryType.LogsVolume:

‎src/features/log/LogsModel.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,7 @@ export function queryLogsVolume<TQuery extends DataQuery, TOptions extends DataS
7979
observer.complete();
8080
},
8181
next: (dataQueryResponse: DataQueryResponse) => {
82+
console.log('dataQueryResponse', dataQueryResponse);
8283
const { errors } = dataQueryResponse;
8384
if (errors !== undefined) {
8485
observer.next({

‎src/features/log/queryResponseBuilder.ts‎

Lines changed: 7 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,6 @@ export const getLogsDataFrame = (
1414
const logsData = getDefaultDataFrame(target.refId, 'logs');
1515

1616
logsData.addField({
17-
config: {
18-
filterable: true,
19-
},
2017
name: 'Time',
2118
type: FieldType.time,
2219
});
@@ -39,12 +36,7 @@ export const getLogsDataFrame = (
3936
return logsData;
4037
};
4138

42-
export const getGraphDataFrame = (
43-
data: any,
44-
target: MyQuery,
45-
app: string,
46-
timestampColumn = '_timestamp'
47-
) => {
39+
export const getGraphDataFrame = (data: any, target: MyQuery, app: string, timestampColumn = '_timestamp') => {
4840
const graphData = getDefaultDataFrame(target.refId, 'graph');
4941

5042
let fields = ['zo_sql_key', 'zo_sql_num'];
@@ -69,6 +61,7 @@ export const getGraphDataFrame = (
6961
} else {
7062
graphData.addField({
7163
name: fields[i],
64+
type: FieldType.number,
7265
});
7366
}
7467
}
@@ -78,7 +71,7 @@ export const getGraphDataFrame = (
7871
}
7972

8073
data.forEach((log: any) => {
81-
graphData.add(getField(log, fields, timestampColumn));
74+
graphData.add(getField(log, fields, 'zo_sql_key'));
8275
});
8376

8477
return graphData;
@@ -89,16 +82,17 @@ const getField = (log: any, columns: any, timestampColumn: string) => {
8982

9083
for (let i = 0; i < columns.length; i++) {
9184
let col_name = columns[i];
92-
let col_value = log[col_name]
85+
let col_value = log[col_name];
9386
if (isTimeField(col_name, timestampColumn)) {
9487
// We have to convert microseconds if we receive them
9588
// 500 billion / year 17814 is probably a good threshold for milliseconds
89+
9690
if (col_value > 500_000_000_000) {
9791
col_value = convertTimeToMs(col_value);
98-
field["Time"] = col_value;
92+
field['Time'] = col_value;
9993
} else {
10094
// Convert any other date fmt
101-
field["Time"] = new Date(col_value).getTime();
95+
field['Time'] = new Date(col_value + 'Z').getTime();
10296
}
10397
} else {
10498
field[col_name] = log[col_name];

‎src/features/query/queryBuilder.ts‎

Lines changed: 3 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -17,55 +17,8 @@ export const buildQuery = (
1717
start_time: timestamps.startTimeInMicro,
1818
end_time: timestamps.endTimeInMirco,
1919
size: 300,
20-
},
21-
aggs: {
22-
histogram: `select histogram(${timestampColumn}, '[INTERVAL]') AS zo_sql_key, count(*) AS zo_sql_num from query GROUP BY zo_sql_key ORDER BY zo_sql_key`,
23-
},
24-
};
25-
26-
if (timestamps.startTimeInMicro && timestamps.endTimeInMirco) {
27-
req.query.start_time = timestamps.startTimeInMicro;
28-
req.query.end_time = timestamps.endTimeInMirco;
29-
30-
let chartInterval = '1 second';
31-
32-
const timeDifference = (timestamps.endTimeInMirco - timestamps.startTimeInMicro) / 1000;
33-
34-
if (timeDifference >= 1000 * 60 * 5) {
35-
chartInterval = '3 second';
36-
}
37-
if (timeDifference >= 1000 * 60 * 10) {
38-
chartInterval = '5 second';
39-
}
40-
if (timeDifference >= 1000 * 60 * 20) {
41-
chartInterval = '10 second';
42-
}
43-
if (timeDifference >= 1000 * 60 * 30) {
44-
chartInterval = '15 second';
45-
}
46-
if (timeDifference >= 1000 * 60 * 60) {
47-
chartInterval = '30 second';
48-
}
49-
if (timeDifference >= 1000 * 3600 * 2) {
50-
chartInterval = '1 minute';
5120
}
52-
if (timeDifference >= 1000 * 3600 * 6) {
53-
chartInterval = '5 minute';
54-
}
55-
if (timeDifference >= 1000 * 3600 * 24) {
56-
chartInterval = '30 minute';
57-
}
58-
if (timeDifference >= 1000 * 86400 * 7) {
59-
chartInterval = '1 hour';
60-
}
61-
if (timeDifference >= 1000 * 86400 * 30) {
62-
chartInterval = '1 day';
63-
}
64-
65-
req.aggs.histogram = req.aggs.histogram.replaceAll('[INTERVAL]', chartInterval);
66-
} else {
67-
return false;
68-
}
21+
};
6922

7023
if (app !== 'explore') {
7124
req.query.size = 0;
@@ -74,7 +27,6 @@ export const buildQuery = (
7427
if (queryData.sqlMode) {
7528
req.query.sql = queryData.query;
7629
req.query['sql_mode'] = 'full';
77-
delete req.aggs;
7830
}
7931

8032
if (!queryData.sqlMode) {
@@ -112,11 +64,8 @@ export const buildQuery = (
11264
req.query.sql = req.query.sql.replace('[INDEX_NAME]', queryData.stream);
11365
}
11466

115-
req['encoding'] = 'base64';
116-
req.query.sql = b64EncodeUnicode(req.query.sql);
117-
if (!queryData.sqlMode) {
118-
req.aggs.histogram = b64EncodeUnicode(req.aggs.histogram);
119-
}
67+
// req['encoding'] = 'base64';
68+
// req.query.sql = b64EncodeUnicode(req.query.sql);
12069

12170
return req;
12271
} catch (e) {

0 commit comments

Comments
 (0)