Skip to content

Commit 5786c7e

Browse files
richie-niKartheeswaran Subramaniam
richie-ni
and
Kartheeswaran Subramaniam
authored
feat(dataframe): Include associated data table from results (#712)
Co-authored-by: Kartheeswaran Subramaniam <kartheeswaran.subramaniam@emerson.com>
1 parent 4087980 commit 5786c7e

File tree

4 files changed

+263
-38
lines changed

4 files changed

+263
-38
lines changed

‎src/datasources/data-frame/constants.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ export const COLUMN_OPTIONS_LIMIT = 10000;
3636
export const COLUMN_SELECTION_LIMIT = 20;
3737
export const MAXIMUM_DATA_POINTS = 1000000;
3838
export const RESULT_IDS_LIMIT = 1000;
39+
export const DATA_TABLES_IDS_LIMIT = 1000;
3940
export const CUSTOM_PROPERTY_COLUMNS_LIMIT = 100;
4041
export const CUSTOM_PROPERTY_OPTIONS_LIMIT = 10_000;
4142

‎src/datasources/data-frame/datasources/v2/DataFrameDataSourceV2.test.ts‎

Lines changed: 140 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ import { DataFrameDataSourceV2 } from './DataFrameDataSourceV2';
22
import { DataQueryRequest, DataSourceInstanceSettings, FieldDTO } from '@grafana/data';
33
import { BackendSrv, locationService, TemplateSrv } from '@grafana/runtime';
44
import { ColumnType, DATA_TABLE_ID_FIELD, DATA_TABLE_NAME_FIELD, DataFrameDataQuery, DataFrameFeatureTogglesDefaults, DataFrameQueryType, DataFrameQueryV1, DataFrameQueryV2, DataFrameVariableQuery, DataFrameVariableQueryType, DataFrameVariableQueryV2, DataTableProjectionLabelLookup, DataTableProjections, DataTableProperties, defaultQueryV2, ValidDataFrameQueryV2 } from '../../types';
5-
import { COLUMN_SELECTION_LIMIT, CUSTOM_COLUMN_PROPERTIES_GROUP, CUSTOM_DATA_TABLE_PROPERTIES_GROUP, MAXIMUM_DATA_POINTS, propertiesCacheTTL, REQUESTS_PER_SECOND, TAKE_LIMIT } from 'datasources/data-frame/constants';
5+
import { COLUMN_SELECTION_LIMIT, CUSTOM_COLUMN_PROPERTIES_GROUP, CUSTOM_DATA_TABLE_PROPERTIES_GROUP, DATA_TABLES_IDS_LIMIT, MAXIMUM_DATA_POINTS, propertiesCacheTTL, REQUESTS_PER_SECOND, TAKE_LIMIT } from 'datasources/data-frame/constants';
66
import * as queryBuilderUtils from 'core/query-builder.utils';
77
import { DataTableQueryBuilderFieldNames } from 'datasources/data-frame/components/v2/constants/DataTableQueryBuilder.constants';
88
import { Workspace } from 'core/types';
@@ -9633,7 +9633,7 @@ describe('DataFrameDataSourceV2', () => {
96339633
`${instanceSettings.url}/nitestmonitor/v2/query-results`,
96349634
{
96359635
filter: 'status = "Passed"',
9636-
projection: ['id'],
9636+
projection: ['id', 'dataTableIds'],
96379637
take: 1000,
96389638
orderBy: 'UPDATED_AT',
96399639
descending: true
@@ -9680,6 +9680,7 @@ describe('DataFrameDataSourceV2', () => {
96809680

96819681
const result = await lastValueFrom(ds.queryTables$(filters, 10));
96829682

9683+
expect(postMock$).toHaveBeenCalledTimes(1);
96839684
expect(postMock$).toHaveBeenCalledWith(
96849685
`${ds.baseUrl}/query-tables`,
96859686
{
@@ -9709,7 +9710,7 @@ describe('DataFrameDataSourceV2', () => {
97099710
`${instanceSettings.url}/nitestmonitor/v2/query-results`,
97109711
{
97119712
filter: 'status = "Passed"',
9712-
projection: ['id'],
9713+
projection: ['id', 'dataTableIds'],
97139714
take: 1000,
97149715
orderBy: 'UPDATED_AT',
97159716
descending: true
@@ -9996,6 +9997,142 @@ describe('DataFrameDataSourceV2', () => {
99969997
],
99979998
});
99989999
});
10000+
10001+
describe('when data table ids are associated with results', () => {
10002+
beforeEach(() => {
10003+
postMock$.mockImplementation((url) => {
10004+
if (!url.includes('query-results')) {
10005+
return of({ tables: mockTables });
10006+
}
10007+
10008+
return of({
10009+
results: [
10010+
{
10011+
id: 'result-1',
10012+
dataTableIds: [
10013+
'dt-1',
10014+
'dt-2'
10015+
]
10016+
},
10017+
{
10018+
id: 'result-2',
10019+
dataTableIds: [
10020+
'dt-3'
10021+
]
10022+
}
10023+
]
10024+
});
10025+
});
10026+
});
10027+
10028+
it('should include result IDs and associated data table IDs from results in query tables filter with substitutions', async () => {
10029+
const filters = {
10030+
resultFilter: 'status = "Passed"',
10031+
dataTableFilter: ''
10032+
};
10033+
await lastValueFrom(ds.queryTables$(filters));
10034+
10035+
expect(postMock$).toHaveBeenCalledWith(
10036+
`${ds.baseUrl}/query-tables`,
10037+
{
10038+
interactive: true,
10039+
orderBy: 'ROWS_MODIFIED_AT',
10040+
orderByDescending: true,
10041+
filter: '((new[]{@0,@1}.Contains(testResultId))||(new[]{@2,@3,@4}.Contains(id)))',
10042+
take: TAKE_LIMIT,
10043+
projection: [DataTableProjections.RowsModifiedAt],
10044+
substitutions: [
10045+
'result-1',
10046+
'result-2',
10047+
'dt-1',
10048+
'dt-2',
10049+
'dt-3'
10050+
]
10051+
},
10052+
{ useApiIngress: true, showErrorAlert: false }
10053+
);
10054+
});
10055+
10056+
it('should limit associated data table IDs from results to 1000', async () => {
10057+
const dataTableIds = Array.from(
10058+
{ length: 1500 },
10059+
(_, index) => `dt-${index}`
10060+
);
10061+
postMock$.mockImplementation((url) => {
10062+
if (url.includes('query-results')) {
10063+
return of({
10064+
results: [
10065+
{
10066+
id: 'result-1',
10067+
dataTableIds: dataTableIds.slice(0, 750)
10068+
},
10069+
{
10070+
id: 'result-2',
10071+
dataTableIds: dataTableIds.slice(750, 1500)
10072+
}
10073+
]
10074+
});
10075+
}
10076+
return of({ tables: mockTables });
10077+
});
10078+
const expectedResultIds = ['result-1', 'result-2'];
10079+
const expectedDataTableIds = dataTableIds.slice(0, DATA_TABLES_IDS_LIMIT);
10080+
const expectedSubstitutions = [...expectedResultIds, ...expectedDataTableIds];
10081+
const resultIdPlaceholders = expectedResultIds.map(
10082+
(_, index) => `@${index}`
10083+
).join(',');
10084+
const dataTableIdPlaceholders = expectedDataTableIds.map(
10085+
(_, index) => `@${expectedResultIds.length + index}`
10086+
).join(',');
10087+
10088+
const filters = { resultFilter: 'status = "Passed"', dataTableFilter: '' };
10089+
await lastValueFrom(ds.queryTables$(filters));
10090+
10091+
expect(postMock$).toHaveBeenCalledWith(
10092+
`${ds.baseUrl}/query-tables`,
10093+
{
10094+
interactive: true,
10095+
orderBy: 'ROWS_MODIFIED_AT',
10096+
orderByDescending: true,
10097+
filter: `((new[]{${resultIdPlaceholders}}.Contains(testResultId))||(new[]{${dataTableIdPlaceholders}}.Contains(id)))`,
10098+
take: TAKE_LIMIT,
10099+
projection: [DataTableProjections.RowsModifiedAt],
10100+
substitutions: expectedSubstitutions
10101+
},
10102+
{ useApiIngress: true, showErrorAlert: false }
10103+
);
10104+
});
10105+
10106+
it('should combine result filter, data table filter and column filter with AND when provided', async () => {
10107+
const filters = {
10108+
resultFilter: 'status = "Passed"',
10109+
dataTableFilter: 'name = "Table1"',
10110+
columnFilter: 'columns.any(it.name = "Column1")'
10111+
};
10112+
10113+
await lastValueFrom(ds.queryTables$(filters, 10));
10114+
10115+
expect(postMock$).toHaveBeenCalledWith(
10116+
`${ds.baseUrl}/query-tables`,
10117+
{
10118+
interactive: true,
10119+
orderBy: 'ROWS_MODIFIED_AT',
10120+
orderByDescending: true,
10121+
filter: '((new[]{@0,@1}.Contains(testResultId))||(new[]{@2,@3,@4}.Contains(id)))&&(name = "Table1")&&(columns.any(it.name = "Column1"))',
10122+
take: 10,
10123+
projection: [DataTableProjections.RowsModifiedAt],
10124+
substitutions: [
10125+
'result-1',
10126+
'result-2',
10127+
'dt-1',
10128+
'dt-2',
10129+
'dt-3'
10130+
]
10131+
},
10132+
{ useApiIngress: true, showErrorAlert: false }
10133+
);
10134+
});
10135+
});
999910136
});
1000010137

1000110138
describe('getColumnOptionsWithVariables', () => {

‎src/datasources/data-frame/datasources/v2/DataFrameDataSourceV2.ts‎

Lines changed: 115 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
import { AppEvents, createDataFrame, DataFrameDTO, DataQueryRequest, DataSourceInstanceSettings, dateTime, FieldDTO, FieldType, LegacyMetricFindQueryOptions, MetricFindValue, QueryResultMetaNotice, ScopedVars, TimeRange } from "@grafana/data";
22
import { DataFrameDataSourceBase } from "../../DataFrameDataSourceBase";
33
import { BackendSrv, getBackendSrv, TemplateSrv, getTemplateSrv } from "@grafana/runtime";
4-
import { Column, Option, DataFrameDataQuery, DataFrameDataSourceOptions, DataFrameQueryType, DataFrameQueryV2, DataFrameVariableQuery, DataFrameVariableQueryType, DataTableProjectionLabelLookup, DataTableProjections, DataTableProperties, defaultQueryV2, defaultVariableQueryV2, FlattenedTableProperties, TableDataRows, TableProperties, TablePropertiesList, ValidDataFrameQueryV2, ValidDataFrameVariableQuery, DataFrameQueryV1, DecimatedDataRequest, UndecimatedDataRequest, ColumnFilter, CombinedFilters, QueryResultsResponse, ColumnOptions, ColumnType, TableColumnsData, ColumnWithDisplayName, ColumnDataType, metadataFieldOptions, DATA_TABLE_NAME_FIELD, DATA_TABLE_ID_FIELD, DATA_TABLE_NAME_LABEL, DATA_TABLE_ID_LABEL, CustomPropertyOptions, PropertySelections, ALL_STANDARD_PROPERTIES, PropertiesQueryCache } from "../../types";
5-
import { COLUMN_OPTIONS_LIMIT, COLUMN_SELECTION_LIMIT, COLUMNS_GROUP, CUSTOM_PROPERTY_COLUMNS_LIMIT, DELAY_BETWEEN_REQUESTS_MS, FLOAT32_MAX, FLOAT32_MIN, FLOAT64_MAX, FLOAT64_MIN, INT32_MAX, INT32_MIN, INT64_MAX, INT64_MIN, X_COLUMN_RANGE_DECIMAL_PRECISION, INTEGER_DATA_TYPES, NUMERIC_DATA_TYPES, POSSIBLE_UNIT_CUSTOM_PROPERTY_KEYS, REQUESTS_PER_SECOND, RESULT_IDS_LIMIT, TAKE_LIMIT, MAXIMUM_DATA_POINTS, UNDECIMATED_RECORDS_LIMIT, CUSTOM_COLUMN_PROPERTIES_GROUP, CUSTOM_DATA_TABLE_PROPERTIES_GROUP, CUSTOM_PROPERTY_SUFFIX, propertiesCacheTTL } from "datasources/data-frame/constants";
4+
import { Column, Option, DataFrameDataQuery, DataFrameDataSourceOptions, DataFrameQueryType, DataFrameQueryV2, DataFrameVariableQuery, DataFrameVariableQueryType, DataTableProjectionLabelLookup, DataTableProjections, DataTableProperties, defaultQueryV2, defaultVariableQueryV2, FlattenedTableProperties, TableDataRows, TableProperties, TablePropertiesList, ValidDataFrameQueryV2, ValidDataFrameVariableQuery, DataFrameQueryV1, DecimatedDataRequest, UndecimatedDataRequest, ColumnFilter, CombinedFilters, DataFrameQueryResultsResponse, ColumnOptions, ColumnType, TableColumnsData, ColumnWithDisplayName, ColumnDataType, metadataFieldOptions, DATA_TABLE_NAME_FIELD, DATA_TABLE_ID_FIELD, DATA_TABLE_NAME_LABEL, DATA_TABLE_ID_LABEL, CustomPropertyOptions, PropertySelections, ALL_STANDARD_PROPERTIES, PropertiesQueryCache, DataFrameResultsResponseProperties } from "../../types";
5+
import { COLUMN_OPTIONS_LIMIT, COLUMN_SELECTION_LIMIT, COLUMNS_GROUP, CUSTOM_PROPERTY_COLUMNS_LIMIT, DELAY_BETWEEN_REQUESTS_MS, FLOAT32_MAX, FLOAT32_MIN, FLOAT64_MAX, FLOAT64_MIN, INT32_MAX, INT32_MIN, INT64_MAX, INT64_MIN, X_COLUMN_RANGE_DECIMAL_PRECISION, INTEGER_DATA_TYPES, NUMERIC_DATA_TYPES, POSSIBLE_UNIT_CUSTOM_PROPERTY_KEYS, REQUESTS_PER_SECOND, RESULT_IDS_LIMIT, TAKE_LIMIT, MAXIMUM_DATA_POINTS, UNDECIMATED_RECORDS_LIMIT, CUSTOM_COLUMN_PROPERTIES_GROUP, CUSTOM_DATA_TABLE_PROPERTIES_GROUP, CUSTOM_PROPERTY_SUFFIX, propertiesCacheTTL, DATA_TABLES_IDS_LIMIT } from "datasources/data-frame/constants";
66
import { ExpressionTransformFunction, listFieldsQuery, multipleValuesQuery, timeFieldsQuery, transformComputedFieldsQuery } from "core/query-builder.utils";
77
import { LEGACY_METADATA_TYPE, Workspace } from "core/types";
88
import { extractErrorInfo } from "core/errors";
@@ -196,28 +196,52 @@ export class DataFrameDataSourceV2 extends DataFrameDataSourceBase {
196196
take?: number,
197197
projections?: DataTableProjections[]
198198
): Observable<TableProperties[]> {
199-
if (filters.resultFilter) {
200-
return this.queryResultIds$(filters.resultFilter).pipe(
201-
switchMap(resultIds => {
202-
if (resultIds.length === 0) {
203-
return of([]);
204-
}
205-
const resultFilter = this.buildResultIdFilter(resultIds);
206-
const combinedFilter = this.buildCombinedFilter({
207-
resultFilter,
208-
dataTableFilter: filters.dataTableFilter,
209-
columnFilter: filters.columnFilter
210-
});
211-
return this.queryTablesInternal$(
212-
combinedFilter,
213-
take,
214-
projections,
215-
resultIds
216-
);
217-
})
218-
);
199+
const filterAndSubstitutions$ = this.buildQueryTablesFilterAndSubstitutions$(filters);
200+
return filterAndSubstitutions$.pipe(
201+
switchMap(filterAndSubstitutions => {
202+
if (!filterAndSubstitutions) {
203+
return of([]);
204+
}
205+
206+
return this.queryTablesInternal$(
207+
filterAndSubstitutions.filter,
208+
take,
209+
projections,
210+
filterAndSubstitutions.substitutions
211+
);
212+
})
213+
);
214+
}
215+
216+
private buildQueryTablesFilterAndSubstitutions$(
217+
filters: CombinedFilters
218+
): Observable<{ filter: string; substitutions?: string[] } | null> {
219+
if (!filters.resultFilter) {
220+
return of({
221+
filter: filters.dataTableFilter || ''
222+
});
219223
}
220-
return this.queryTablesInternal$(filters.dataTableFilter || '', take, projections, undefined);
224+
225+
const results$ = this.queryResults$(filters.resultFilter);
226+
return results$.pipe(
227+
map(results => {
228+
if (results.length === 0) {
229+
return null;
230+
}
231+
232+
const {
233+
filter: resultFilter,
234+
substitutions
235+
} = this.buildResultFilter(results);
236+
const filter = this.buildCombinedFilter({
237+
resultFilter,
238+
dataTableFilter: filters.dataTableFilter,
239+
columnFilter: filters.columnFilter
240+
});
241+
242+
return { filter, substitutions };
243+
})
244+
);
221245
}
222246

223247
private queryTablesInternal$(
@@ -2219,17 +2243,19 @@ export class DataFrameDataSourceV2 extends DataFrameDataSourceBase {
22192243
return propertyKeysSet;
22202244
}
22212245

2222-
private queryResultIds$(resultFilter: string): Observable<string[]> {
2246+
private queryResults$(
2247+
resultFilter: string
2248+
): Observable<DataFrameResultsResponseProperties[]> {
22232249
const queryResultsUrl = `${this.instanceSettings.url}/nitestmonitor/v2/query-results`;
22242250
const requestBody = {
22252251
filter: resultFilter,
2226-
projection: ['id'],
2252+
projection: ['id', 'dataTableIds'],
22272253
take: RESULT_IDS_LIMIT,
22282254
orderBy: 'UPDATED_AT',
22292255
descending: true
22302256
};
22312257

2232-
return this.post$<QueryResultsResponse>(
2258+
return this.post$<DataFrameQueryResultsResponse>(
22332259
queryResultsUrl,
22342260
requestBody,
22352261
{ showErrorAlert: false }
@@ -2238,7 +2264,7 @@ export class DataFrameDataSourceV2 extends DataFrameDataSourceBase {
22382264
if (!response.results || response.results.length === 0) {
22392265
return [];
22402266
}
2241-
return response.results.map(result => result.id);
2267+
return response.results;
22422268
}),
22432269
catchError(error => {
22442270
const errorMessage = this.getErrorMessage(error, 'results');
@@ -2251,13 +2277,51 @@ export class DataFrameDataSourceV2 extends DataFrameDataSourceBase {
22512277
);
22522278
}
22532279

2254-
private buildResultIdFilter(resultIds: string[]): string {
2255-
if (resultIds.length === 0) {
2256-
return '';
2280+
private buildResultFilter(
2281+
results: DataFrameResultsResponseProperties[]
2282+
): { filter: string; substitutions?: string[] } {
2283+
if (results.length === 0) {
2284+
return { filter: '' };
22572285
}
2258-
const placeholders = resultIds.map((_, index) => `@${index}`).join(',');
2259-
const resultFilter = `new[]{${placeholders}}.Contains(testResultId)`;
2260-
return resultFilter;
2286+
2287+
const {
2288+
resultIds,
2289+
dataTableIds
2290+
} = this.extractResultAndDataTableIds(results);
2291+
2292+
const resultIdFilter = this.buildPlaceholderContainsFilter(
2293+
'testResultId',
2294+
resultIds.length,
2295+
0
2296+
);
2297+
if (dataTableIds.length === 0) {
2298+
return {
2299+
filter: resultIdFilter,
2300+
substitutions: resultIds
2301+
};
2302+
}
2303+
2304+
const dataTableIdFilter = this.buildPlaceholderContainsFilter(
2305+
'id',
2306+
dataTableIds.length,
2307+
resultIds.length
2308+
);
2309+
return {
2310+
filter: `(${resultIdFilter})||(${dataTableIdFilter})`,
2311+
substitutions: [...resultIds, ...dataTableIds]
2312+
};
2313+
}
2314+
2315+
private buildPlaceholderContainsFilter(
2316+
fieldName: string,
2317+
count: number,
2318+
startIndex: number
2319+
): string {
2320+
const placeholders = Array.from(
2321+
{ length: count },
2322+
(_, index) => `@${startIndex + index}`
2323+
).join(',');
2324+
return `new[]{${placeholders}}.Contains(${fieldName})`;
22612325
}
22622326

22632327
private buildCombinedFilter(filters: CombinedFilters): string {
@@ -2334,4 +2398,22 @@ export class DataFrameDataSourceV2 extends DataFrameDataSourceBase {
23342398

23352399
return defaultQueryV2.filterXRangeOnZoomPan;
23362400
}
2401+
2402+
private extractResultAndDataTableIds(
2403+
results: DataFrameResultsResponseProperties[]
2404+
): { resultIds: string[]; dataTableIds: string[] } {
2405+
const resultIds: string[] = [];
2406+
const dataTableIdSet = new Set<string>();
2407+
2408+
for (const { id, dataTableIds } of results) {
2409+
resultIds.push(id);
2410+
dataTableIds?.forEach(dataTableId => {
2411+
if (dataTableIdSet.size < DATA_TABLES_IDS_LIMIT) {
2412+
dataTableIdSet.add(dataTableId);
2413+
}
2414+
});
2415+
}
2416+
2417+
return { resultIds, dataTableIds: [...dataTableIdSet] };
2418+
}
23372419
}

‎src/datasources/data-frame/types.ts‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -145,8 +145,13 @@ export const defaultQueryV2: Omit<ValidDataFrameQueryV2, 'refId'> = {
145145
undecimatedRecordCount: 10_000
146146
};
147147

148-
export interface QueryResultsResponse {
149-
results: Array<{id: string}>;
148+
export interface DataFrameQueryResultsResponse {
149+
results: DataFrameResultsResponseProperties[];
150+
}
151+
152+
export interface DataFrameResultsResponseProperties {
153+
id: string;
154+
dataTableIds?: string[];
150155
}
151156

152157
export const DataTableProjectionLabelLookup: Record<DataTableProperties, {

0 commit comments

Comments
 (0)