Skip to content

Commit e61ead2

Browse files
XiaJunjie2020XiaJunjie2020
committed
feat(datasource): commit any code for excel
1 parent 0851d8d commit e61ead2

File tree

4 files changed

+108
-43
lines changed

4 files changed

+108
-43
lines changed

‎backend/apps/datasource/api/datasource.py‎

Lines changed: 26 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,14 @@
1-
from fastapi import APIRouter, File, UploadFile
1+
from fastapi import APIRouter, File, UploadFile, HTTPException
22
from ..crud.datasource import get_datasource_list, check_status, create_ds, update_ds, delete_ds, getTables, getFields, \
33
execSql, update_table_and_fields, getTablesByDs, chooseTables, preview
44
from common.core.deps import SessionDep
55
from ..models.datasource import CoreDatasource, CreateDatasource, EditObj, CoreTable
66
from ..crud.table import get_tables_by_ds_id
77
from ..crud.field import get_fields_by_table_id
88
from typing import List
9+
import uuid
10+
import pandas as pd
11+
import hashlib
912

1013
router = APIRouter(tags=["datasource"], prefix="/datasource")
1114

@@ -80,7 +83,27 @@ async def edit_local(session: SessionDep, id: int, data: EditObj):
8083
return preview(session, id, data)
8184

8285

83-
@router.post("/uploadExcel/")
86+
@router.post("/uploadExcel")
8487
async def upload_excel(session: SessionDep, file: UploadFile = File(...)):
88+
ALLOWED_EXTENSIONS = {"xlsx", "xls", "csv"}
89+
if not file.filename.lower().endswith(tuple(ALLOWED_EXTENSIONS)):
90+
raise HTTPException(400, "Only support .xlsx/.xls/.csv")
91+
92+
contents = await file.read()
93+
df_sheets = pd.read_excel(contents, sheet_name=None)
94+
# build columns and data to insert db
8595
# todo
86-
pass
96+
sheets = []
97+
for sheet_name, df in df_sheets.items():
98+
print("--------------------")
99+
print(f"Sheet: {sheet_name}, {hashlib.sha256(uuid.uuid4().bytes).hexdigest()[:10]}")
100+
sheets.append({"name": f"{sheet_name}",
101+
"uniqueName": f"{sheet_name}_{hashlib.sha256(uuid.uuid4().bytes).hexdigest()[:10]}"})
102+
column_len = len(df.dtypes)
103+
for i in range(column_len):
104+
print(f"{df.columns[i]} , {df.dtypes[i]}")
105+
106+
for row in df.values:
107+
print(row)
108+
109+
return {"filename": file.filename, "sheets": sheets}

‎backend/apps/db/engine.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
# Author: Junjun
2+
# Date: 2025/5/19

‎backend/pyproject.toml‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,9 @@ dependencies = [
2929
"pymysql (>=1.1.1,<2.0.0)",
3030
"cryptography (>=44.0.3,<45.0.0)",
3131
"llama_index>=0.12.35",
32-
"pymssql (>=2.3.4,<3.0.0)"
32+
"pymssql (>=2.3.4,<3.0.0)",
33+
"pandas (>=2.2.3,<3.0.0)",
34+
"openpyxl (>=3.1.5,<4.0.0)"
3335
]
3436
[[tool.uv.index]]
3537
url = "https://pypi.tuna.tsinghua.edu.cn/simple"

‎frontend/src/views/ds/form.vue‎

Lines changed: 77 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -29,28 +29,47 @@
2929
/>
3030
</el-select>
3131
</el-form-item>
32-
<el-form-item label="Host/Ip">
33-
<el-input v-model="config.host" />
34-
</el-form-item>
35-
<el-form-item label="Port">
36-
<el-input v-model="config.port" />
37-
</el-form-item>
38-
<el-form-item label="Username">
39-
<el-input v-model="config.username" />
40-
</el-form-item>
41-
<el-form-item label="Password">
42-
<el-input v-model="config.password" type="password" show-password />
43-
</el-form-item>
44-
<el-form-item label="Database">
45-
<el-input v-model="config.database" />
46-
</el-form-item>
47-
<el-form-item label="Extra JDBC String">
48-
<el-input v-model="config.extraJdbc" />
49-
</el-form-item>
50-
<el-form-item label="Schema" v-if="form.type === 'sqlServer'">
51-
<el-input v-model="config.dbSchema" />
52-
<el-button link type="primary" :icon="Plus" v-if="false">Get Schema</el-button>
53-
</el-form-item>
32+
<div v-if="form.type === 'excel'">
33+
<el-form-item label="File">
34+
<el-upload
35+
accept=".xls, .xlsx, .csv"
36+
:headers="headers"
37+
action="http://localhost:8000/api/v1/datasource/uploadExcel"
38+
:before-upload="beforeUpload"
39+
>
40+
<el-button>Upload</el-button>
41+
<template #tip>
42+
<div class="el-upload__tip">
43+
Only support .xls, .xlsx, .csv, size less than 50MB.
44+
</div>
45+
</template>
46+
</el-upload>
47+
</el-form-item>
48+
</div>
49+
<div v-else>
50+
<el-form-item label="Host/Ip">
51+
<el-input v-model="config.host" />
52+
</el-form-item>
53+
<el-form-item label="Port">
54+
<el-input v-model="config.port" />
55+
</el-form-item>
56+
<el-form-item label="Username">
57+
<el-input v-model="config.username" />
58+
</el-form-item>
59+
<el-form-item label="Password">
60+
<el-input v-model="config.password" type="password" show-password />
61+
</el-form-item>
62+
<el-form-item label="Database">
63+
<el-input v-model="config.database" />
64+
</el-form-item>
65+
<el-form-item label="Extra JDBC String">
66+
<el-input v-model="config.extraJdbc" />
67+
</el-form-item>
68+
<el-form-item label="Schema" v-if="form.type === 'sqlServer'">
69+
<el-input v-model="config.dbSchema" />
70+
<el-button link type="primary" :icon="Plus" v-if="false">Get Schema</el-button>
71+
</el-form-item>
72+
</div>
5473
</el-form>
5574
</div>
5675
<div v-show="active === 2" class="container">
@@ -67,7 +86,7 @@
6786
</div>
6887
<div style="display: flex;justify-content: flex-end;margin-top: 20px;">
6988
<el-button @click="close">Cancel</el-button>
70-
<el-button v-show="!isCreate && !isEditTable" @click="check">Test Connect</el-button>
89+
<el-button v-show="!isCreate && !isEditTable && form.type !== 'excel'" @click="check">Test Connect</el-button>
7190
<el-button v-show="active === 1 && isCreate" type="primary" @click="next(dsFormRef)">Next</el-button>
7291
<el-button v-show="active === 2 && isCreate" @click="preview">Preview</el-button>
7392
<el-button v-show="active === 2 || !isCreate" type="primary" @click="save(dsFormRef)">Save</el-button>
@@ -81,7 +100,9 @@ import { encrypted, decrypted } from './js/aes'
81100
import { ElMessage } from 'element-plus'
82101
import type { FormInstance, FormRules } from 'element-plus'
83102
import { Plus } from '@element-plus/icons-vue'
103+
import { useCache } from '@/utils/useCache'
84104
105+
const { wsCache } = useCache()
85106
const dsFormRef = ref<FormInstance>()
86107
const emit = defineEmits(['refresh'])
87108
const active = ref(1)
@@ -90,6 +111,10 @@ const isEditTable = ref(false)
90111
const checkList = ref<any>([])
91112
const tableList = ref<any>([])
92113
114+
const token = wsCache.get('user.token')
115+
const headers = ref<any>({'X-SQLBOT-TOKEN': `Bearer ${token}`})
116+
117+
93118
const rules = reactive<FormRules>({
94119
name: [
95120
{ required: true, message: 'Please input name', trigger: 'blur' },
@@ -99,6 +124,7 @@ const rules = reactive<FormRules>({
99124
100125
const dialogVisible = ref<boolean>(false)
101126
const dsType = [
127+
{label:"Excel/CSV", value:"excel"},
102128
{label:"MySQL", value:"mysql"},
103129
{label:"SQL Server", value:"sqlServer"}
104130
]
@@ -246,28 +272,40 @@ const check = () => {
246272
}
247273
248274
const next = async(formEl: FormInstance | undefined) => {
249-
if(!formEl) return
250-
await formEl.validate((valid) => {
251-
if (valid) {
252-
// check status if success do next
253-
buildConf()
254-
datasourceApi.check(form.value).then((res: boolean) => {
255-
if(res) {
256-
active.value++
257-
// request tables
258-
datasourceApi.getTablesByConf(form.value).then((res) => {
259-
tableList.value = res
260-
})
261-
}
262-
})
263-
}
264-
})
275+
if (form.value.type === "excel") {
276+
// next, show tables
277+
} else {
278+
if(!formEl) return
279+
await formEl.validate((valid) => {
280+
if (valid) {
281+
// check status if success do next
282+
buildConf()
283+
datasourceApi.check(form.value).then((res: boolean) => {
284+
if(res) {
285+
active.value++
286+
// request tables
287+
datasourceApi.getTablesByConf(form.value).then((res) => {
288+
tableList.value = res
289+
})
290+
}
291+
})
292+
}
293+
})
294+
}
265295
}
266296
267297
const preview = () => {
268298
active.value--
269299
}
270300
301+
const beforeUpload = (rawFile: any) => {
302+
if (rawFile.size / 1024 / 1024 > 50) {
303+
ElMessage.error('File size can not exceed 50MB!')
304+
return false
305+
}
306+
return true
307+
}
308+
271309
defineExpose({ open })
272310
</script>
273311
<style lang="less" scoped>

0 commit comments

Comments
 (0)