Skip to content

Commit 1277c7c

Browse files
BlankllBlankll
authored
feat: implement dynamodb pagination (#217)
feat: implement dynamodb pagination Refs: #142 --------- Signed-off-by: seven <[email protected]>
1 parent 0f0a519 commit 1277c7c

File tree

8 files changed

+296
-61
lines changed

8 files changed

+296
-61
lines changed

‎package-lock.json‎

Lines changed: 5 additions & 4 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@
6565
"eslint-plugin-vue": "^10.0.0",
6666
"husky": "^9.1.7",
6767
"jest": "^29.7.0",
68-
"naive-ui": "^2.41.0",
68+
"naive-ui": "^2.42.0",
6969
"prettier": "^3.5.3",
7070
"sass": "^1.86.3",
7171
"ts-jest": "^29.3.2",

‎src-tauri/src/dynamo/query_table.rs‎

Lines changed: 36 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,12 +25,46 @@ pub async fn query_table(
2525
let pk_name = partition_key.get("name").and_then(|v| v.as_str()).unwrap();
2626
let pk_value = partition_key.get("value").and_then(|v| v.as_str()).unwrap();
2727

28-
let mut query = client.query().table_name(input.table_name);
28+
let limit = input
29+
.payload
30+
.get("limit")
31+
.and_then(|v| v.as_u64())
32+
.map(|v| v as i32)
33+
.unwrap();
34+
let exclusive_start_key = input
35+
.payload
36+
.get("exclusive_start_key")
37+
.and_then(|v| v.as_object());
38+
39+
let mut query = client.query().table_name(input.table_name).limit(limit);
2940

3041
if let Some(idx_name) = index_name {
3142
query = query.index_name(idx_name);
3243
}
3344

45+
if let Some(key_map) = exclusive_start_key {
46+
let mut last_key_map = std::collections::HashMap::new();
47+
48+
for (k, v) in key_map {
49+
if let Some(s) = v.as_str() {
50+
last_key_map.insert(k.clone(), AttributeValue::S(s.to_string()));
51+
} else if let Some(n) = v.as_u64() {
52+
last_key_map.insert(k.clone(), AttributeValue::N(n.to_string()));
53+
} else if let Some(n) = v.as_i64() {
54+
last_key_map.insert(k.clone(), AttributeValue::N(n.to_string()));
55+
} else if let Some(b) = v.as_bool() {
56+
last_key_map.insert(k.clone(), AttributeValue::Bool(b));
57+
}
58+
}
59+
60+
if !last_key_map.is_empty() {
61+
// Apply each key-value pair individually to match the method signature
62+
for (key, value) in last_key_map {
63+
query = query.exclusive_start_key(key, value);
64+
}
65+
}
66+
}
67+
3468
// Attribute name placeholders
3569
let mut expr_attr_names = Vec::<(String, String)>::new();
3670
expr_attr_names.push(("#attr0".to_string(), pk_name.to_string()));
@@ -174,7 +208,7 @@ pub async fn query_table(
174208
message: "Query executed successfully".to_string(),
175209
data: Some(json!({
176210
"items": json_items,
177-
"count": items.len(),
211+
"count": response.count(),
178212
"scanned_count": response.scanned_count(),
179213
"last_evaluated_key": match response.last_evaluated_key() {
180214
Some(key_map) => {

‎src-tauri/src/dynamo/scan_table.rs‎

Lines changed: 35 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,14 +11,47 @@ pub struct ScanTableInput<'a> {
1111
pub async fn scan_table(client: &Client, input: ScanTableInput<'_>) -> Result<ApiResponse, String> {
1212
let index_name = input.payload.get("index_name").and_then(|v| v.as_str());
1313
let filters = input.payload.get("filters").and_then(|v| v.as_array());
14+
let limit = input
15+
.payload
16+
.get("limit")
17+
.and_then(|v| v.as_u64())
18+
.map(|v| v as i32)
19+
.unwrap();
20+
let exclusive_start_key = input
21+
.payload
22+
.get("exclusive_start_key")
23+
.and_then(|v| v.as_object());
1424

1525
// Start building the scan
16-
let mut scan = client.scan().table_name(input.table_name);
26+
let mut scan = client.scan().table_name(input.table_name).limit(limit);
1727

1828
if let Some(idx_name) = index_name {
1929
scan = scan.index_name(idx_name);
2030
}
2131

32+
if let Some(key_map) = exclusive_start_key {
33+
let mut last_key_map = std::collections::HashMap::new();
34+
35+
for (k, v) in key_map {
36+
if let Some(s) = v.as_str() {
37+
last_key_map.insert(k.clone(), AttributeValue::S(s.to_string()));
38+
} else if let Some(n) = v.as_u64() {
39+
last_key_map.insert(k.clone(), AttributeValue::N(n.to_string()));
40+
} else if let Some(n) = v.as_i64() {
41+
last_key_map.insert(k.clone(), AttributeValue::N(n.to_string()));
42+
} else if let Some(b) = v.as_bool() {
43+
last_key_map.insert(k.clone(), AttributeValue::Bool(b));
44+
}
45+
}
46+
47+
if !last_key_map.is_empty() {
48+
// Apply each key-value pair individually to match the method signature
49+
for (key, value) in last_key_map {
50+
scan = scan.exclusive_start_key(key, value);
51+
}
52+
}
53+
}
54+
2255
// Add filters if provided
2356
if let Some(filter_array) = filters {
2457
let mut filter_expressions = Vec::new();
@@ -163,7 +196,7 @@ pub async fn scan_table(client: &Client, input: ScanTableInput<'_>) -> Result<Ap
163196
message: "Scan executed successfully".to_string(),
164197
data: Some(json!({
165198
"items": json_items,
166-
"count": items.len(),
199+
"count": response.count(),
167200
"scanned_count": response.scanned_count(),
168201
"last_evaluated_key": match response.last_evaluated_key() {
169202
Some(key_map) => {

‎src/datasources/dynamoApi.ts‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -68,17 +68,19 @@ export type QueryParams = {
6868
indexName: string | null;
6969
partitionKey: {
7070
name: string;
71-
value: string | null;
71+
value: string | null | undefined;
7272
};
7373
sortKey?: {
7474
name: string;
7575
value: string;
7676
};
77-
filters: Array<{
77+
filters?: Array<{
7878
key: string;
7979
operator: string;
8080
value: string;
8181
}>;
82+
limit?: number; // Optional limit for the number of items to return
83+
exclusiveStartKey?: Record<string, unknown> | null; // For pagination
8284
};
8385

8486
export type QueryResult = {
@@ -143,6 +145,8 @@ const dynamoApi = {
143145
partition_key: queryParams.partitionKey,
144146
sort_key: queryParams.sortKey,
145147
filters: queryParams.filters,
148+
limit: queryParams.limit,
149+
exclusive_start_key: queryParams.exclusiveStartKey,
146150
},
147151
};
148152

@@ -165,7 +169,12 @@ const dynamoApi = {
165169
const options = {
166170
table_name: queryParams.tableName,
167171
operation: 'SCAN_TABLE',
168-
payload: { filters: queryParams.filters, index_name: queryParams.indexName },
172+
payload: {
173+
filters: queryParams.filters,
174+
index_name: queryParams.indexName,
175+
limit: queryParams.limit,
176+
exclusive_start_key: queryParams.exclusiveStartKey,
177+
},
169178
};
170179

171180
const result = await tauriClient.invokeDynamoApi(credentials, options);

‎src/store/dbDataStore.ts‎

Lines changed: 175 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,175 @@
1+
import { defineStore } from 'pinia';
2+
import { cloneDeep, omit } from 'lodash';
3+
import { DynamoDBConnection, useConnectionStore } from './connectionStore.ts';
4+
import { DynamoIndexOrTableOption } from './tabStore.ts';
5+
6+
const resetPagination = {
7+
page: 1,
8+
pageSize: 10,
9+
pageCount: 1,
10+
showSizePicker: true,
11+
pageSizes: [10, 25, 50, 100, 200, 300],
12+
};
13+
14+
type DynamoInput = {
15+
index?: string;
16+
partitionKey?: string;
17+
sortKey?: string;
18+
filters?: Array<{ key: string; value: string; operator: string }>;
19+
};
20+
21+
type DynamoColumn = {
22+
title: string;
23+
key: string;
24+
children?: Array<{ title: string; key: string }>;
25+
};
26+
27+
export const useDbDataStore = defineStore('dbDataStore', {
28+
state: (): {
29+
dynamoData: {
30+
connection: DynamoDBConnection;
31+
columns: Array<DynamoColumn>;
32+
data: Array<Record<string, unknown>> | undefined;
33+
pagination: {
34+
page: number;
35+
pageSize: number;
36+
pageCount: number;
37+
showSizePicker: boolean;
38+
pageSizes: Array<number>;
39+
};
40+
queryInput?: DynamoInput;
41+
queryBody: string;
42+
lastEvaluatedKeys: Array<Record<string, any>>;
43+
};
44+
} => ({
45+
dynamoData: {
46+
connection: {} as DynamoDBConnection,
47+
columns: [],
48+
data: undefined,
49+
pagination: cloneDeep(resetPagination),
50+
queryInput: undefined,
51+
queryBody: '',
52+
lastEvaluatedKeys: [],
53+
},
54+
}),
55+
persist: true,
56+
57+
actions: {
58+
async getDynamoData(connection: DynamoDBConnection, queryInput: DynamoInput): Promise<void> {
59+
this.dynamoData.connection = connection;
60+
61+
try {
62+
const { getDynamoIndexOrTableOption, queryTable } = useConnectionStore();
63+
64+
const { tableName } = connection;
65+
const { partitionKey, sortKey } = queryInput;
66+
const indices = getDynamoIndexOrTableOption(connection);
67+
const { partitionKeyName, sortKeyName, label, value } = indices.find(
68+
item => item.label === queryInput.index,
69+
) as DynamoIndexOrTableOption;
70+
71+
const queryParams = {
72+
tableName,
73+
indexName: label.startsWith('Table - ') ? null : (value ?? null),
74+
partitionKey: { name: partitionKeyName, value: partitionKey },
75+
sortKey: sortKeyName && sortKey ? { name: sortKeyName, value: sortKey } : undefined,
76+
filters: queryInput.filters,
77+
};
78+
79+
const queryStr = JSON.stringify(omit(queryParams, ['limit', 'exclusiveStartKey']));
80+
81+
if (this.dynamoData.queryBody !== queryStr) {
82+
this.dynamoData = {
83+
...this.dynamoData,
84+
columns: [],
85+
data: undefined,
86+
pagination: { ...cloneDeep(resetPagination) },
87+
queryBody: queryStr,
88+
lastEvaluatedKeys: [],
89+
};
90+
}
91+
92+
const limit = this.dynamoData.pagination.pageSize;
93+
const exclusiveStartKey =
94+
this.dynamoData.lastEvaluatedKeys[this.dynamoData.pagination.page - 1];
95+
96+
const data = await queryTable(connection, { ...queryParams, limit, exclusiveStartKey });
97+
98+
const columnsSet = new Set<string>();
99+
data.items.forEach(item => {
100+
Object.keys(item).forEach(key => {
101+
columnsSet.add(key);
102+
});
103+
});
104+
105+
const columnsData = data.items.map(item => {
106+
const row: Record<string, unknown> = {};
107+
columnsSet.forEach(key => {
108+
row[key] = item[key];
109+
});
110+
return row;
111+
});
112+
113+
const primaryColumn = {
114+
title: 'Primary Key',
115+
key: 'primaryKey',
116+
children: [
117+
{ title: `${partitionKeyName}(PK)`, key: `${partitionKeyName}` },
118+
sortKeyName ? { title: `${sortKeyName}(SK)`, key: `${sortKeyName}` } : undefined,
119+
].filter(Boolean) as Array<{ title: string; key: string }>,
120+
};
121+
const columns: Array<DynamoColumn> = Array.from(columnsSet)
122+
.filter(column => column !== partitionKeyName && column !== sortKeyName)
123+
.map(column => ({ title: column, key: column }));
124+
columns.unshift(primaryColumn);
125+
126+
this.dynamoData.columns = columns;
127+
this.dynamoData.data = columnsData;
128+
129+
if (data.last_evaluated_key) {
130+
this.dynamoData.lastEvaluatedKeys[this.dynamoData.pagination.page] =
131+
data.last_evaluated_key;
132+
this.dynamoData.pagination.pageCount = this.dynamoData.lastEvaluatedKeys.length;
133+
}
134+
this.dynamoData.queryInput = queryInput;
135+
} catch (error) {
136+
throw error;
137+
}
138+
},
139+
140+
async changePage(page: number) {
141+
if (this.dynamoData.pagination.page !== page) {
142+
this.dynamoData.pagination.page = page;
143+
await this.getDynamoData(
144+
this.dynamoData.connection,
145+
this.dynamoData.queryInput as DynamoInput,
146+
);
147+
}
148+
},
149+
150+
async changePageSize(pageSize: number) {
151+
if (this.dynamoData.pagination.pageSize !== pageSize) {
152+
this.dynamoData.pagination.pageSize = pageSize;
153+
this.dynamoData.pagination.page = 1;
154+
this.dynamoData.pagination.pageCount = 1;
155+
this.dynamoData.lastEvaluatedKeys = [];
156+
await this.getDynamoData(
157+
this.dynamoData.connection,
158+
this.dynamoData.queryInput as DynamoInput,
159+
);
160+
}
161+
},
162+
163+
resetDynamoData() {
164+
this.dynamoData = {
165+
connection: {} as DynamoDBConnection,
166+
columns: [],
167+
data: undefined,
168+
pagination: cloneDeep(resetPagination),
169+
queryInput: undefined,
170+
queryBody: '',
171+
lastEvaluatedKeys: [],
172+
};
173+
},
174+
},
175+
});

‎src/store/index.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,3 +6,4 @@ export * from './chatStore';
66
export * from './clusterManageStore';
77
export * from './backupRestoreStore';
88
export * from './tabStore.ts';
9+
export * from './dbDataStore.ts';

0 commit comments

Comments
 (0)