feat: chart & export Excel support large number data

#233

Author: ulleo

backend/apps/chat/api/chat.py

@@ -2,15 +2,15 @@
 import io
 import traceback
 
-import numpy as np
 import orjson
 import pandas as pd
 from fastapi import APIRouter, HTTPException
 from fastapi.responses import StreamingResponse
 from sqlalchemy import and_, select
 
 from apps.chat.curd.chat import list_chats, get_chat_with_records, create_chat, rename_chat, \
-    delete_chat, get_chat_chart_data, get_chat_predict_data, get_chat_with_records_with_data, get_chat_record_by_id
+    delete_chat, get_chat_chart_data, get_chat_predict_data, get_chat_with_records_with_data, get_chat_record_by_id, \
+    format_json_data
 from apps.chat.models.chat_model import CreateChat, ChatRecord, RenameChat, ChatQuestion, ExcelData
 from apps.chat.task.llm import LLMService
 from common.core.deps import CurrentAssistant, SessionDep, CurrentUser, Trans
@@ -45,15 +45,17 @@ def inner():
 @router.get("/record/get/{chart_record_id}/data")
 async def chat_record_data(session: SessionDep, chart_record_id: int):
     def inner():
-        return get_chat_chart_data(chart_record_id=chart_record_id, session=session)
+        data = get_chat_chart_data(chart_record_id=chart_record_id, session=session)
+        return format_json_data(data)
 
     return await asyncio.to_thread(inner)
 
 
 @router.get("/record/get/{chart_record_id}/predict_data")
 async def chat_predict_data(session: SessionDep, chart_record_id: int):
     def inner():
-        return get_chat_predict_data(chart_record_id=chart_record_id, session=session)
+        data = get_chat_predict_data(chart_record_id=chart_record_id, session=session)
+        return format_json_data(data)
 
     return await asyncio.to_thread(inner)
 
@@ -211,21 +213,53 @@ def inner():
                 detail=trans("i18n_excel_export.data_is_empty")
             )
 
+        # 预处理数据并记录每列的格式类型
+        col_formats = {}  # 格式类型：'text'（文本）、'number'（数字）、'default'（默认）
+        for field_idx, field in enumerate(excel_data.axis):
+            _fields_list.append(field.name)
+            col_formats[field_idx] = 'default'  # 默认不特殊处理
+
         for _data in excel_data.data:
             _row = []
-            for field in excel_data.axis:
-                _row.append(_data.get(field.value))
+            for field_idx, field in enumerate(excel_data.axis):
+                value = _data.get(field.value)
+                if value is not None:
+                    # 检查是否为数字且需要特殊处理
+                    if isinstance(value, (int, float)):
+                        # 整数且超过15位 → 转字符串并标记为文本列
+                        if isinstance(value, int) and len(str(abs(value))) > 15:
+                            value = str(value)
+                            col_formats[field_idx] = 'text'
+                        # 小数且超过15位有效数字 → 转字符串并标记为文本列
+                        elif isinstance(value, float):
+                            decimal_str = format(value, '.16f').rstrip('0').rstrip('.')
+                            if len(decimal_str) > 15:
+                                value = str(value)
+                                col_formats[field_idx] = 'text'
+                        # 其他数字列标记为数字格式（避免科学记数法）
+                        elif col_formats[field_idx] != 'text':
+                            col_formats[field_idx] = 'number'
+                _row.append(value)
             data.append(_row)
-        for field in excel_data.axis:
-            _fields_list.append(field.name)
-        df = pd.DataFrame(np.array(data), columns=_fields_list)
+
+        df = pd.DataFrame(data, columns=_fields_list)
 
         buffer = io.BytesIO()
 
         with pd.ExcelWriter(buffer, engine='xlsxwriter',
-                            engine_kwargs={'options': {'strings_to_numbers': True}}) as writer:
+                            engine_kwargs={'options': {'strings_to_numbers': False}}) as writer:
             df.to_excel(writer, sheet_name='Sheet1', index=False)
 
+            # 获取 xlsxwriter 的工作簿和工作表对象
+            workbook = writer.book
+            worksheet = writer.sheets['Sheet1']
+
+            for col_idx, fmt_type in col_formats.items():
+                if fmt_type == 'text':
+                    worksheet.set_column(col_idx, col_idx, None, workbook.add_format({'num_format': '@'}))
+                elif fmt_type == 'number':
+                    worksheet.set_column(col_idx, col_idx, None, workbook.add_format({'num_format': '0'}))
+
         buffer.seek(0)
         return io.BytesIO(buffer.getvalue())
 

backend/apps/chat/curd/chat.py

@@ -86,6 +86,30 @@ def get_last_execute_sql_error(session: SessionDep, chart_id: int):
     return None
 
 
+def format_json_data(origin_data: dict):
+    result = {'fields': origin_data.get('fields') if origin_data.get('fields') else []}
+    data = []
+    for _data in origin_data.get('data') if origin_data.get('data') else []:
+        _row = {}
+        for key, value in _data.items():
+            if value is not None:
+                # 检查是否为数字且需要特殊处理
+                if isinstance(value, (int, float)):
+                    # 整数且超过15位 → 转字符串并标记为文本列
+                    if isinstance(value, int) and len(str(abs(value))) > 15:
+                        value = str(value)
+                    # 小数且超过15位有效数字 → 转字符串并标记为文本列
+                    elif isinstance(value, float):
+                        decimal_str = format(value, '.16f').rstrip('0').rstrip('.')
+                        if len(decimal_str) > 15:
+                            value = str(value)
+            _row[key] = value
+        data.append(_row)
+    result['data'] = data
+
+    return result
+
+
 def get_chat_chart_data(session: SessionDep, chart_record_id: int):
     stmt = select(ChatRecord.data).where(and_(ChatRecord.id == chart_record_id))
     res = session.execute(stmt)
@@ -94,7 +118,7 @@ def get_chat_chart_data(session: SessionDep, chart_record_id: int):
             return orjson.loads(row.data)
         except Exception:
             pass
-    return []
+    return {}
 
 
 def get_chat_predict_data(session: SessionDep, chart_record_id: int):
@@ -105,7 +129,7 @@ def get_chat_predict_data(session: SessionDep, chart_record_id: int):
             return orjson.loads(row.predict_data)
         except Exception:
             pass
-    return []
+    return {}
 
 
 def get_chat_with_records_with_data(session: SessionDep, chart_id: int, current_user: CurrentUser,

backend/apps/chat/task/llm.py

@@ -8,7 +8,6 @@
 from datetime import datetime
 from typing import Any, List, Optional, Union, Dict, Iterator
 
-import numpy as np
 import orjson
 import pandas as pd
 import requests
@@ -1078,7 +1077,7 @@ def run_task(self, in_chat: bool = True, stream: bool = True,
                         if not data or not _fields_list:
                             yield 'The SQL execution result is empty.\n\n'
                         else:
-                            df = pd.DataFrame(np.array(data), columns=_fields_list)
+                            df = pd.DataFrame(data, columns=_fields_list)
                             markdown_table = df.to_markdown(index=False)
                             yield markdown_table + '\n\n'
                 else:
@@ -1138,7 +1137,7 @@ def run_task(self, in_chat: bool = True, stream: bool = True,
                     if not data or not _fields_list:
                         yield 'The SQL execution result is empty.\n\n'
                     else:
-                        df = pd.DataFrame(np.array(data), columns=_fields_list)
+                        df = pd.DataFrame(data, columns=_fields_list)
                         markdown_table = df.to_markdown(index=False)
                         yield markdown_table + '\n\n'
 

frontend/src/views/chat/component/charts/Bar.ts

@@ -88,6 +88,7 @@ export class Bar extends BaseG2Chart {
         },
         y: {
           nice: true,
+          type: 'linear',
         },
       },
       interaction: {

frontend/src/views/chat/component/charts/Column.ts

@@ -77,6 +77,7 @@ export class Column extends BaseG2Chart {
         },
         y: {
           nice: true,
+          type: 'linear',
         },
       },
       interaction: {

frontend/src/views/chat/component/charts/Line.ts

@@ -51,6 +51,7 @@ export class Line extends BaseG2Chart {
         },
         y: {
           nice: true,
+          type: 'linear',
         },
       },
       children: [

g2-ssr/charts/bar.js

@@ -69,6 +69,7 @@ function getBarOptions(baseOptions, axis, data) {
             },
             y: {
                 nice: true,
+                type: 'linear',
             },
         },
         interaction: {

g2-ssr/charts/column.js

@@ -68,6 +68,7 @@ function getColumnOptions(baseOptions, axis, data) {
             },
             y: {
                 nice: true,
+                type: 'linear',
             },
         },
         interaction: {

g2-ssr/charts/line.js

@@ -42,6 +42,7 @@ function getLineOptions(baseOptions, axis, data) {
             },
             y: {
                 nice: true,
+                type: 'linear',
             },
         },
         children: [