Skip to content

Commit fc56129

Browse files
MikeNitsenkoMikeNitsenko
committed
dev
1 parent 4f836bc commit fc56129

File tree

3 files changed

+332
-1
lines changed

3 files changed

+332
-1
lines changed

‎packages/cubejs-query-orchestrator/examples/datasource-schema-example.ts‎

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
/**
2+
* Example usage of the datasource metadata query methods
3+
*/
4+
5+
import { QuerySchemasResult, QueryTablesResult, QueryColumnsResult } from '@cubejs-backend/base-driver';
6+
import { QueryCache } from '../src/orchestrator/QueryCache';
7+
8+
// Example usage of the new datasource metadata query methods
9+
async function exampleUsage() {
10+
// Assuming you have a QueryCache instance
11+
const queryCache = new QueryCache(
12+
'example',
13+
(_dataSource) => {
14+
// Your driver factory implementation
15+
throw new Error('Driver factory not implemented in example');
16+
},
17+
(msg, params) => console.log(msg, params), // logger
18+
{
19+
cacheAndQueueDriver: 'cubestore', // Use CubeStore for caching
20+
cubeStoreDriverFactory: async () => {
21+
// Your CubeStore driver factory implementation
22+
throw new Error('CubeStore driver factory not implemented in example');
23+
}
24+
}
25+
);
26+
27+
try {
28+
// 1. Query datasource schemas with default options (24h cache)
29+
const schemas: QuerySchemasResult[] = await queryCache.queryDataSourceSchemas('default', {
30+
requestId: 'example-request-1'
31+
});
32+
33+
console.log('Available schemas:', schemas);
34+
// Output: [{ schema_name: 'public' }, { schema_name: 'analytics' }]
35+
36+
// 2. Query tables for specific schemas
37+
const tables: QueryTablesResult[] = await queryCache.queryTablesForSchemas(schemas, 'default', {
38+
requestId: 'example-request-2'
39+
});
40+
41+
console.log('Tables in schemas:', tables);
42+
// Output: [
43+
// { schema_name: 'public', table_name: 'users' },
44+
// { schema_name: 'public', table_name: 'orders' },
45+
// { schema_name: 'analytics', table_name: 'events' }
46+
// ]
47+
48+
// 3. Query columns for specific tables
49+
const columns: QueryColumnsResult[] = await queryCache.queryColumnsForTables(tables, 'default', {
50+
requestId: 'example-request-3'
51+
});
52+
53+
console.log('Columns in tables:', columns);
54+
// Output: [
55+
// { schema_name: 'public', table_name: 'users', column_name: 'id', data_type: 'integer' },
56+
// { schema_name: 'public', table_name: 'users', column_name: 'name', data_type: 'text' },
57+
// // ... more columns
58+
// ]
59+
60+
// 4. Force refresh examples
61+
const freshSchemas = await queryCache.queryDataSourceSchemas('default', {
62+
requestId: 'example-request-4',
63+
forceRefresh: true
64+
});
65+
66+
const _freshTables = await queryCache.queryTablesForSchemas(freshSchemas, 'default', {
67+
requestId: 'example-request-5',
68+
forceRefresh: true,
69+
renewalThreshold: 60 * 60, // 1 hour
70+
expiration: 2 * 24 * 60 * 60 // 2 days
71+
});
72+
73+
// 5. Clear cache examples
74+
await queryCache.clearDataSourceSchemaCache('default');
75+
console.log('Schema cache cleared');
76+
77+
await queryCache.clearTablesForSchemasCache(schemas, 'default');
78+
console.log('Tables cache cleared');
79+
80+
await queryCache.clearColumnsForTablesCache(tables, 'default');
81+
console.log('Columns cache cleared');
82+
83+
// 6. Chain queries for complete metadata discovery
84+
const allSchemas = await queryCache.queryDataSourceSchemas('analytics');
85+
const allTables = await queryCache.queryTablesForSchemas(allSchemas, 'analytics');
86+
const allColumns = await queryCache.queryColumnsForTables(allTables, 'analytics');
87+
88+
console.log('Complete metadata for analytics datasource:');
89+
console.log('Schemas:', allSchemas.length);
90+
console.log('Tables:', allTables.length);
91+
console.log('Columns:', allColumns.length);
92+
} catch (error) {
93+
console.error('Error querying datasource metadata:', error);
94+
}
95+
}
96+
97+
// Example helper function to build a complete metadata tree
98+
async function buildCompleteMetadataTree(
99+
queryCache: QueryCache,
100+
dataSource: string = 'default'
101+
) {
102+
const schemas = await queryCache.queryDataSourceSchemas(dataSource);
103+
const metadataTree: any[] = [];
104+
105+
for (const schema of schemas) {
106+
const tables = await queryCache.queryTablesForSchemas([schema], dataSource);
107+
const schemaNode: any = {
108+
schema_name: schema.schema_name,
109+
tables: []
110+
};
111+
112+
for (const table of tables) {
113+
const columns = await queryCache.queryColumnsForTables([table], dataSource);
114+
schemaNode.tables.push({
115+
table_name: table.table_name,
116+
columns: columns.map(col => ({
117+
column_name: col.column_name,
118+
data_type: col.data_type,
119+
attributes: col.attributes,
120+
foreign_keys: col.foreign_keys
121+
}))
122+
});
123+
}
124+
125+
metadataTree.push(schemaNode);
126+
}
127+
128+
return metadataTree;
129+
}
130+
131+
export { exampleUsage, buildCompleteMetadataTree };

‎packages/cubejs-query-orchestrator/src/orchestrator/QueryCache.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1127,7 +1127,7 @@ export class QueryCache {
11271127
);
11281128
}
11291129

1130-
public async queryDataSourceSchema(
1130+
public async queryDataSourceSchemas(
11311131
dataSource: string = 'default',
11321132
options: {
11331133
requestId?: string;

‎packages/cubejs-query-orchestrator/test/datasource-schema.test.ts‎

Lines changed: 200 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,200 @@
1+
/**
2+
* Comprehensive test for datasource metadata query methods
3+
*/
4+
5+
import { QuerySchemasResult, QueryTablesResult, QueryColumnsResult } from '@cubejs-backend/base-driver';
6+
import { QueryCache } from '../src/orchestrator/QueryCache';
7+
8+
// Mock driver that returns sample metadata
9+
class MockDriver {
10+
public async getSchemas(): Promise<QuerySchemasResult[]> {
11+
await new Promise(resolve => setTimeout(resolve, 100));
12+
return [
13+
{ schema_name: 'public' },
14+
{ schema_name: 'analytics' },
15+
{ schema_name: 'reporting' }
16+
];
17+
}
18+
19+
public async getTablesForSpecificSchemas(schemas: QuerySchemasResult[]): Promise<QueryTablesResult[]> {
20+
await new Promise(resolve => setTimeout(resolve, 100));
21+
const tables: QueryTablesResult[] = [];
22+
23+
schemas.forEach(schema => {
24+
if (schema.schema_name === 'public') {
25+
tables.push(
26+
{ schema_name: 'public', table_name: 'users' },
27+
{ schema_name: 'public', table_name: 'orders' }
28+
);
29+
} else if (schema.schema_name === 'analytics') {
30+
tables.push(
31+
{ schema_name: 'analytics', table_name: 'events' },
32+
{ schema_name: 'analytics', table_name: 'metrics' }
33+
);
34+
}
35+
});
36+
37+
return tables;
38+
}
39+
40+
public async getColumnsForSpecificTables(tables: QueryTablesResult[]): Promise<QueryColumnsResult[]> {
41+
await new Promise(resolve => setTimeout(resolve, 100));
42+
const columns: QueryColumnsResult[] = [];
43+
44+
tables.forEach(table => {
45+
if (table.table_name === 'users') {
46+
columns.push(
47+
{ schema_name: 'public', table_name: 'users', column_name: 'id', data_type: 'integer' },
48+
{ schema_name: 'public', table_name: 'users', column_name: 'name', data_type: 'text' },
49+
{ schema_name: 'public', table_name: 'users', column_name: 'email', data_type: 'text' }
50+
);
51+
} else if (table.table_name === 'orders') {
52+
columns.push(
53+
{ schema_name: 'public', table_name: 'orders', column_name: 'id', data_type: 'integer' },
54+
{ schema_name: 'public', table_name: 'orders', column_name: 'user_id', data_type: 'integer' },
55+
{ schema_name: 'public', table_name: 'orders', column_name: 'amount', data_type: 'decimal' }
56+
);
57+
} else if (table.table_name === 'events') {
58+
columns.push(
59+
{ schema_name: 'analytics', table_name: 'events', column_name: 'id', data_type: 'bigint' },
60+
{ schema_name: 'analytics', table_name: 'events', column_name: 'event_type', data_type: 'text' },
61+
{ schema_name: 'analytics', table_name: 'events', column_name: 'timestamp', data_type: 'timestamp' }
62+
);
63+
}
64+
});
65+
66+
return columns;
67+
}
68+
}// Mock cache driver
69+
class MockCacheDriver {
70+
private cache = new Map<string, any>();
71+
72+
public async get(key: string) {
73+
return this.cache.get(key) || null;
74+
}
75+
76+
public async set(key: string, value: any, _expiration: number) {
77+
this.cache.set(key, value);
78+
return { bytes: JSON.stringify(value).length };
79+
}
80+
81+
public async remove(key: string) {
82+
this.cache.delete(key);
83+
}
84+
85+
public async cleanup() {
86+
this.cache.clear();
87+
}
88+
89+
public async testConnection() {
90+
return true;
91+
}
92+
93+
public async withLock(key: string, callback: any) {
94+
return callback();
95+
}
96+
}
97+
98+
async function testDatasourceMetadataMethods() {
99+
const mockLogger = (msg: string, params?: any) => {
100+
console.log(`[TEST] ${msg}`, params || '');
101+
};
102+
103+
const queryCache = new QueryCache(
104+
'test',
105+
() => Promise.resolve(new MockDriver() as any),
106+
mockLogger,
107+
{
108+
cacheAndQueueDriver: 'memory',
109+
queueOptions: () => Promise.resolve({
110+
concurrency: 1,
111+
continueWaitTimeout: 1000,
112+
executionTimeout: 30000,
113+
})
114+
}
115+
);
116+
117+
// Replace the cache driver with our mock
118+
(queryCache as any).cacheDriver = new MockCacheDriver();
119+
120+
console.log('Testing datasource metadata methods...');
121+
122+
try {
123+
// Test 1: Query schemas
124+
console.log('\n--- Test 1: Query schemas (cache miss) ---');
125+
const schemas = await queryCache.queryDataSourceSchemas('default', {
126+
requestId: 'test-schemas-1'
127+
});
128+
console.log('Schemas:', schemas);
129+
130+
// Test 2: Query tables for schemas
131+
console.log('\n--- Test 2: Query tables for schemas ---');
132+
const tables = await queryCache.queryTablesForSchemas(schemas.slice(0, 2), 'default', {
133+
requestId: 'test-tables-1'
134+
});
135+
console.log('Tables:', tables);
136+
137+
// Test 3: Query columns for tables
138+
console.log('\n--- Test 3: Query columns for tables ---');
139+
const columns = await queryCache.queryColumnsForTables(tables.slice(0, 2), 'default', {
140+
requestId: 'test-columns-1'
141+
});
142+
console.log('Columns:', columns);
143+
144+
// Test 4: Cache hit scenarios
145+
console.log('\n--- Test 4: Cache hits ---');
146+
const cachedSchemas = await queryCache.queryDataSourceSchemas('default', {
147+
requestId: 'test-schemas-2'
148+
});
149+
console.log('Cached schemas count:', cachedSchemas.length);
150+
151+
const cachedTables = await queryCache.queryTablesForSchemas(schemas.slice(0, 2), 'default', {
152+
requestId: 'test-tables-2'
153+
});
154+
console.log('Cached tables count:', cachedTables.length);
155+
156+
// Test 5: Force refresh
157+
console.log('\n--- Test 5: Force refresh ---');
158+
const freshSchemas = await queryCache.queryDataSourceSchemas('default', {
159+
requestId: 'test-schemas-3',
160+
forceRefresh: true
161+
});
162+
console.log('Fresh schemas count:', freshSchemas.length);
163+
164+
// Test 6: Cache clearing
165+
console.log('\n--- Test 6: Cache clearing ---');
166+
await queryCache.clearDataSourceSchemaCache('default');
167+
await queryCache.clearTablesForSchemasCache(schemas.slice(0, 2), 'default');
168+
await queryCache.clearColumnsForTablesCache(tables.slice(0, 2), 'default');
169+
console.log('All caches cleared');
170+
171+
// Test 7: Queries after cache clear
172+
console.log('\n--- Test 7: Queries after cache clear ---');
173+
const newSchemas = await queryCache.queryDataSourceSchemas('default', {
174+
requestId: 'test-schemas-4'
175+
});
176+
console.log('New schemas count:', newSchemas.length);
177+
178+
// Test 8: Full metadata chain
179+
console.log('\n--- Test 8: Full metadata chain ---');
180+
const allSchemas = await queryCache.queryDataSourceSchemas('analytics');
181+
const allTables = await queryCache.queryTablesForSchemas(allSchemas, 'analytics');
182+
const allColumns = await queryCache.queryColumnsForTables(allTables, 'analytics');
183+
184+
console.log('Analytics metadata:');
185+
console.log('- Schemas:', allSchemas.length);
186+
console.log('- Tables:', allTables.length);
187+
console.log('- Columns:', allColumns.length);
188+
189+
console.log('\n✅ All tests passed!');
190+
} catch (error) {
191+
console.error('❌ Test failed:', error);
192+
}
193+
}
194+
195+
// Run the test if this file is executed directly
196+
if (require.main === module) {
197+
testDatasourceMetadataMethods().catch(console.error);
198+
}
199+
200+
export { testDatasourceMetadataMethods };

0 commit comments

Comments
 (0)