Skip to content

Commit 684ad5a

Browse files
authored
feat(tool): added introspection tools for all db integrations (#2780)
* feat(tool): added introspection tools for all db integrations * added sanitization for sql schema * ack pr comments, with clarifying comments * moved util
1 parent a3dff10 commit 684ad5a

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

53 files changed

+3503
-19
lines changed

apps/docs/content/docs/en/tools/dynamodb.mdx

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ This integration empowers Sim agents to automate data management tasks within yo
3737

3838
## Usage Instructions
3939

40-
Integrate Amazon DynamoDB into workflows. Supports Get, Put, Query, Scan, Update, and Delete operations on DynamoDB tables.
40+
Integrate Amazon DynamoDB into workflows. Supports Get, Put, Query, Scan, Update, Delete, and Introspect operations on DynamoDB tables.
4141

4242

4343

@@ -185,6 +185,27 @@ Delete an item from a DynamoDB table
185185
| --------- | ---- | ----------- |
186186
| `message` | string | Operation status message |
187187

188+
### `dynamodb_introspect`
189+
190+
Introspect DynamoDB to list tables or get detailed schema information for a specific table
191+
192+
#### Input
193+
194+
| Parameter | Type | Required | Description |
195+
| --------- | ---- | -------- | ----------- |
196+
| `region` | string | Yes | AWS region \(e.g., us-east-1\) |
197+
| `accessKeyId` | string | Yes | AWS access key ID |
198+
| `secretAccessKey` | string | Yes | AWS secret access key |
199+
| `tableName` | string | No | Optional table name to get detailed schema. If not provided, lists all tables. |
200+
201+
#### Output
202+
203+
| Parameter | Type | Description |
204+
| --------- | ---- | ----------- |
205+
| `message` | string | Operation status message |
206+
| `tables` | array | List of table names in the region |
207+
| `tableDetails` | object | Detailed schema information for a specific table |
208+
188209

189210

190211
## Notes

apps/docs/content/docs/en/tools/elasticsearch.mdx

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -362,6 +362,29 @@ Get comprehensive statistics about the Elasticsearch cluster.
362362
| `nodes` | object | Node statistics including count and versions |
363363
| `indices` | object | Index statistics including document count and store size |
364364

365+
### `elasticsearch_list_indices`
366+
367+
List all indices in the Elasticsearch cluster with their health, status, and statistics.
368+
369+
#### Input
370+
371+
| Parameter | Type | Required | Description |
372+
| --------- | ---- | -------- | ----------- |
373+
| `deploymentType` | string | Yes | Deployment type: self_hosted or cloud |
374+
| `host` | string | No | Elasticsearch host URL \(for self-hosted\) |
375+
| `cloudId` | string | No | Elastic Cloud ID \(for cloud deployments\) |
376+
| `authMethod` | string | Yes | Authentication method: api_key or basic_auth |
377+
| `apiKey` | string | No | Elasticsearch API key |
378+
| `username` | string | No | Username for basic auth |
379+
| `password` | string | No | Password for basic auth |
380+
381+
#### Output
382+
383+
| Parameter | Type | Description |
384+
| --------- | ---- | ----------- |
385+
| `message` | string | Summary message about the indices |
386+
| `indices` | json | Array of index information objects |
387+
365388

366389

367390
## Notes

apps/docs/content/docs/en/tools/google_drive.mdx

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -96,13 +96,13 @@ Download a file from Google Drive with complete metadata (exports Google Workspa
9696
| `fileId` | string | Yes | The ID of the file to download |
9797
| `mimeType` | string | No | The MIME type to export Google Workspace files to \(optional\) |
9898
| `fileName` | string | No | Optional filename override |
99-
| `includeRevisions` | boolean | No | Whether to include revision history in the metadata \(default: true\) |
99+
| `includeRevisions` | boolean | No | Whether to include revision history in the metadata \(default: true, returns first 100 revisions\) |
100100

101101
#### Output
102102

103103
| Parameter | Type | Description |
104104
| --------- | ---- | ----------- |
105-
| `file` | object | Downloaded file stored in execution files |
105+
| `file` | object | Downloaded file data |
106106

107107
### `google_drive_list`
108108

apps/docs/content/docs/en/tools/mongodb.mdx

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -172,6 +172,30 @@ Execute MongoDB aggregation pipeline
172172
| `documents` | array | Array of documents returned from aggregation |
173173
| `documentCount` | number | Number of documents returned |
174174

175+
### `mongodb_introspect`
176+
177+
Introspect MongoDB database to list databases, collections, and indexes
178+
179+
#### Input
180+
181+
| Parameter | Type | Required | Description |
182+
| --------- | ---- | -------- | ----------- |
183+
| `host` | string | Yes | MongoDB server hostname or IP address |
184+
| `port` | number | Yes | MongoDB server port \(default: 27017\) |
185+
| `database` | string | No | Database name to introspect \(optional - if not provided, lists all databases\) |
186+
| `username` | string | No | MongoDB username |
187+
| `password` | string | No | MongoDB password |
188+
| `authSource` | string | No | Authentication database |
189+
| `ssl` | string | No | SSL connection mode \(disabled, required, preferred\) |
190+
191+
#### Output
192+
193+
| Parameter | Type | Description |
194+
| --------- | ---- | ----------- |
195+
| `message` | string | Operation status message |
196+
| `databases` | array | Array of database names |
197+
| `collections` | array | Array of collection info with name, type, document count, and indexes |
198+
175199

176200

177201
## Notes

apps/docs/content/docs/en/tools/mysql.mdx

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -157,6 +157,29 @@ Execute raw SQL query on MySQL database
157157
| `rows` | array | Array of rows returned from the query |
158158
| `rowCount` | number | Number of rows affected |
159159

160+
### `mysql_introspect`
161+
162+
Introspect MySQL database schema to retrieve table structures, columns, and relationships
163+
164+
#### Input
165+
166+
| Parameter | Type | Required | Description |
167+
| --------- | ---- | -------- | ----------- |
168+
| `host` | string | Yes | MySQL server hostname or IP address |
169+
| `port` | number | Yes | MySQL server port \(default: 3306\) |
170+
| `database` | string | Yes | Database name to connect to |
171+
| `username` | string | Yes | Database username |
172+
| `password` | string | Yes | Database password |
173+
| `ssl` | string | No | SSL connection mode \(disabled, required, preferred\) |
174+
175+
#### Output
176+
177+
| Parameter | Type | Description |
178+
| --------- | ---- | ----------- |
179+
| `message` | string | Operation status message |
180+
| `tables` | array | Array of table schemas with columns, keys, and indexes |
181+
| `databases` | array | List of available databases on the server |
182+
160183

161184

162185
## Notes

apps/docs/content/docs/en/tools/neo4j.mdx

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -168,6 +168,33 @@ Execute arbitrary Cypher queries on Neo4j graph database for complex operations
168168
| `recordCount` | number | Number of records returned |
169169
| `summary` | json | Execution summary with timing and counters |
170170

171+
### `neo4j_introspect`
172+
173+
Introspect a Neo4j database to discover its schema including node labels, relationship types, properties, constraints, and indexes.
174+
175+
#### Input
176+
177+
| Parameter | Type | Required | Description |
178+
| --------- | ---- | -------- | ----------- |
179+
| `host` | string | Yes | Neo4j server hostname or IP address |
180+
| `port` | number | Yes | Neo4j server port \(default: 7687 for Bolt protocol\) |
181+
| `database` | string | Yes | Database name to connect to |
182+
| `username` | string | Yes | Neo4j username |
183+
| `password` | string | Yes | Neo4j password |
184+
| `encryption` | string | No | Connection encryption mode \(enabled, disabled\) |
185+
186+
#### Output
187+
188+
| Parameter | Type | Description |
189+
| --------- | ---- | ----------- |
190+
| `message` | string | Operation status message |
191+
| `labels` | array | Array of node labels in the database |
192+
| `relationshipTypes` | array | Array of relationship types in the database |
193+
| `nodeSchemas` | array | Array of node schemas with their properties |
194+
| `relationshipSchemas` | array | Array of relationship schemas with their properties |
195+
| `constraints` | array | Array of database constraints |
196+
| `indexes` | array | Array of database indexes |
197+
171198

172199

173200
## Notes

apps/docs/content/docs/en/tools/postgresql.mdx

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -157,6 +157,30 @@ Execute raw SQL query on PostgreSQL database
157157
| `rows` | array | Array of rows returned from the query |
158158
| `rowCount` | number | Number of rows affected |
159159

160+
### `postgresql_introspect`
161+
162+
Introspect PostgreSQL database schema to retrieve table structures, columns, and relationships
163+
164+
#### Input
165+
166+
| Parameter | Type | Required | Description |
167+
| --------- | ---- | -------- | ----------- |
168+
| `host` | string | Yes | PostgreSQL server hostname or IP address |
169+
| `port` | number | Yes | PostgreSQL server port \(default: 5432\) |
170+
| `database` | string | Yes | Database name to connect to |
171+
| `username` | string | Yes | Database username |
172+
| `password` | string | Yes | Database password |
173+
| `ssl` | string | No | SSL connection mode \(disabled, required, preferred\) |
174+
| `schema` | string | No | Schema to introspect \(default: public\) |
175+
176+
#### Output
177+
178+
| Parameter | Type | Description |
179+
| --------- | ---- | ----------- |
180+
| `message` | string | Operation status message |
181+
| `tables` | array | Array of table schemas with columns, keys, and indexes |
182+
| `schemas` | array | List of available schemas in the database |
183+
160184

161185

162186
## Notes

apps/docs/content/docs/en/tools/rds.mdx

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -165,6 +165,32 @@ Execute raw SQL on Amazon RDS using the Data API
165165
| `rows` | array | Array of rows returned or affected |
166166
| `rowCount` | number | Number of rows affected |
167167

168+
### `rds_introspect`
169+
170+
Introspect Amazon RDS Aurora database schema to retrieve table structures, columns, and relationships
171+
172+
#### Input
173+
174+
| Parameter | Type | Required | Description |
175+
| --------- | ---- | -------- | ----------- |
176+
| `region` | string | Yes | AWS region \(e.g., us-east-1\) |
177+
| `accessKeyId` | string | Yes | AWS access key ID |
178+
| `secretAccessKey` | string | Yes | AWS secret access key |
179+
| `resourceArn` | string | Yes | ARN of the Aurora DB cluster |
180+
| `secretArn` | string | Yes | ARN of the Secrets Manager secret containing DB credentials |
181+
| `database` | string | No | Database name \(optional\) |
182+
| `schema` | string | No | Schema to introspect \(default: public for PostgreSQL, database name for MySQL\) |
183+
| `engine` | string | No | Database engine \(aurora-postgresql or aurora-mysql\). Auto-detected if not provided. |
184+
185+
#### Output
186+
187+
| Parameter | Type | Description |
188+
| --------- | ---- | ----------- |
189+
| `message` | string | Operation status message |
190+
| `engine` | string | Detected database engine type |
191+
| `tables` | array | Array of table schemas with columns, keys, and indexes |
192+
| `schemas` | array | List of available schemas in the database |
193+
168194

169195

170196
## Notes

apps/docs/content/docs/en/tools/supabase.mdx

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -261,6 +261,25 @@ Call a PostgreSQL function in Supabase
261261
| `message` | string | Operation status message |
262262
| `results` | json | Result returned from the function |
263263

264+
### `supabase_introspect`
265+
266+
Introspect Supabase database schema to get table structures, columns, and relationships
267+
268+
#### Input
269+
270+
| Parameter | Type | Required | Description |
271+
| --------- | ---- | -------- | ----------- |
272+
| `projectId` | string | Yes | Your Supabase project ID \(e.g., jdrkgepadsdopsntdlom\) |
273+
| `schema` | string | No | Database schema to introspect \(defaults to all user schemas, commonly "public"\) |
274+
| `apiKey` | string | Yes | Your Supabase service role secret key |
275+
276+
#### Output
277+
278+
| Parameter | Type | Description |
279+
| --------- | ---- | ----------- |
280+
| `message` | string | Operation status message |
281+
| `tables` | array | Array of table schemas with columns, keys, and indexes |
282+
264283
### `supabase_storage_upload`
265284

266285
Upload a file to a Supabase storage bucket
Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
import { randomUUID } from 'crypto'
2+
import { createLogger } from '@sim/logger'
3+
import { type NextRequest, NextResponse } from 'next/server'
4+
import { z } from 'zod'
5+
import { createRawDynamoDBClient, describeTable, listTables } from '@/app/api/tools/dynamodb/utils'
6+
7+
const logger = createLogger('DynamoDBIntrospectAPI')
8+
9+
const IntrospectSchema = z.object({
10+
region: z.string().min(1, 'AWS region is required'),
11+
accessKeyId: z.string().min(1, 'AWS access key ID is required'),
12+
secretAccessKey: z.string().min(1, 'AWS secret access key is required'),
13+
tableName: z.string().optional(),
14+
})
15+
16+
export async function POST(request: NextRequest) {
17+
const requestId = randomUUID().slice(0, 8)
18+
19+
try {
20+
const body = await request.json()
21+
const params = IntrospectSchema.parse(body)
22+
23+
logger.info(`[${requestId}] Introspecting DynamoDB in region ${params.region}`)
24+
25+
const client = createRawDynamoDBClient({
26+
region: params.region,
27+
accessKeyId: params.accessKeyId,
28+
secretAccessKey: params.secretAccessKey,
29+
})
30+
31+
try {
32+
const { tables } = await listTables(client)
33+
34+
if (params.tableName) {
35+
logger.info(`[${requestId}] Describing table: ${params.tableName}`)
36+
const { tableDetails } = await describeTable(client, params.tableName)
37+
38+
logger.info(`[${requestId}] Table description completed for '${params.tableName}'`)
39+
40+
return NextResponse.json({
41+
message: `Table '${params.tableName}' described successfully.`,
42+
tables,
43+
tableDetails,
44+
})
45+
}
46+
47+
logger.info(`[${requestId}] Listed ${tables.length} tables`)
48+
49+
return NextResponse.json({
50+
message: `Found ${tables.length} table(s) in region '${params.region}'.`,
51+
tables,
52+
})
53+
} finally {
54+
client.destroy()
55+
}
56+
} catch (error) {
57+
if (error instanceof z.ZodError) {
58+
logger.warn(`[${requestId}] Invalid request data`, { errors: error.errors })
59+
return NextResponse.json(
60+
{ error: 'Invalid request data', details: error.errors },
61+
{ status: 400 }
62+
)
63+
}
64+
65+
const errorMessage = error instanceof Error ? error.message : 'Unknown error occurred'
66+
logger.error(`[${requestId}] DynamoDB introspection failed:`, error)
67+
68+
return NextResponse.json(
69+
{ error: `DynamoDB introspection failed: ${errorMessage}` },
70+
{ status: 500 }
71+
)
72+
}
73+
}

0 commit comments

Comments
 (0)