Skip to content

Commit 0555632

Browse files
wxg0103wxg0103
committed
feat: add token usage and top questions statistics endpoints
1 parent 74b1bce commit 0555632

File tree

9 files changed

+135
-22
lines changed

9 files changed

+135
-22
lines changed

‎apps/application/flow/step_node/video_understand_step_node/impl/base_video_understand_node.py‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,7 @@
11
# coding=utf-8
2-
import base64
3-
import mimetypes
2+
43
import time
54
from functools import reduce
6-
from imghdr import what
75
from typing import List, Dict
86

97
from django.db.models import QuerySet
@@ -12,7 +10,6 @@
1210
from application.flow.i_step_node import NodeResult, INode
1311
from application.flow.step_node.video_understand_step_node.i_video_understand_node import IVideoUnderstandNode
1412
from knowledge.models import File
15-
from models_provider.impl.volcanic_engine_model_provider.model.image import get_video_format
1613
from models_provider.tools import get_model_instance_by_model_workspace_id
1714

1815

‎apps/application/serializers/application_stats.py‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -118,3 +118,37 @@ def get_days_between_dates(start_date, end_date):
118118
days.append(current_date.strftime('%Y-%m-%d'))
119119
current_date += datetime.timedelta(days=1)
120120
return days
121+
122+
def get_token_usage_statistics(self, with_valid=True):
123+
if with_valid:
124+
self.is_valid(raise_exception=True)
125+
start_time = self.get_start_time()
126+
end_time = self.get_end_time()
127+
get_token_usage = native_search(
128+
{'default_sql': QuerySet(model=get_dynamics_model(
129+
{'application_chat.application_id': models.UUIDField(),
130+
'application_chat_record.create_time': models.DateTimeField()})).filter(
131+
**{'application_chat.application_id': self.data.get('application_id'),
132+
'application_chat_record.create_time__gte': start_time,
133+
'application_chat_record.create_time__lte': end_time}
134+
)},
135+
select_string=get_file_content(
136+
os.path.join(PROJECT_DIR, "apps", "application", 'sql', 'get_token_usage.sql')))
137+
return get_token_usage
138+
139+
def get_top_questions_statistics(self, with_valid=True):
140+
if with_valid:
141+
self.is_valid(raise_exception=True)
142+
start_time = self.get_start_time()
143+
end_time = self.get_end_time()
144+
get_top_questions = native_search(
145+
{'default_sql': QuerySet(model=get_dynamics_model(
146+
{'application_chat.application_id': models.UUIDField(),
147+
'application_chat_record.create_time': models.DateTimeField()})).filter(
148+
**{'application_chat.application_id': self.data.get('application_id'),
149+
'application_chat_record.create_time__gte': start_time,
150+
'application_chat_record.create_time__lte': end_time}
151+
)},
152+
select_string=get_file_content(
153+
os.path.join(PROJECT_DIR, "apps", "application", 'sql', 'top_questions.sql')))
154+
return get_top_questions

‎apps/application/sql/get_token_usage.sql‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
2+
SELECT
3+
SUM(application_chat_record.message_tokens + application_chat_record.answer_tokens) as "token_usage",
4+
COALESCE(application_chat.asker->>'username', '游客') as "username"
5+
FROM
6+
application_chat_record application_chat_record
7+
LEFT JOIN application_chat application_chat ON application_chat."id" = application_chat_record.chat_id
8+
${default_sql}
9+
GROUP BY
10+
COALESCE(application_chat.asker->>'username', '游客')
11+
ORDER BY
12+
"token_usage" DESC

‎apps/application/sql/top_questions.sql‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
SELECT COUNT(application_chat_record."id") AS chat_record_count,
2+
COALESCE(application_chat.asker ->>'username', '游客') AS username
3+
FROM application_chat_record application_chat_record
4+
LEFT JOIN application_chat application_chat ON application_chat."id" = application_chat_record.chat_id
5+
${default_sql}
6+
GROUP BY
7+
COALESCE (application_chat.asker->>'username', '游客')
8+
ORDER BY
9+
chat_record_count DESC,
10+
username ASC
11+

‎apps/application/urls.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@
1313
path('workspace/<str:workspace_id>/application/<str:application_id>/publish', views.ApplicationAPI.Publish.as_view()),
1414
path('workspace/<str:workspace_id>/application/<str:application_id>/application_key', views.ApplicationKey.as_view()),
1515
path('workspace/<str:workspace_id>/application/<str:application_id>/application_stats', views.ApplicationStats.as_view()),
16+
path('workspace/<str:workspace_id>/application/<str:application_id>/application_token_usage', views.ApplicationStats.TokenUsageStatistics.as_view()),
17+
path('workspace/<str:workspace_id>/application/<str:application_id>/top_questions', views.ApplicationStats.TopQuestionsStatistics.as_view()),
1618
path('workspace/<str:workspace_id>/application/<str:application_id>/application_key/<str:api_key_id>', views.ApplicationKey.Operate.as_view()),
1719
path('workspace/<str:workspace_id>/application/<str:application_id>/export', views.ApplicationAPI.Export.as_view()),
1820
path('workspace/<str:workspace_id>/application/<str:application_id>/application_version', views.ApplicationVersionView.as_view()),

‎apps/application/views/application_stats.py‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,3 +46,58 @@ def get(self, request: Request, workspace_id: str, application_id: str):
4646
'end_time': request.query_params.get(
4747
'end_time')
4848
}).get_chat_record_aggregate_trend())
49+
50+
class TokenUsageStatistics(APIView):
51+
authentication_classes = [TokenAuth]
52+
53+
# 应用的token使用统计 根据人的使用数排序
54+
@extend_schema(
55+
methods=['GET'],
56+
description=_('Application token usage statistics'),
57+
summary=_('Application token usage statistics'),
58+
operation_id=_('Application token usage statistics'), # type: ignore
59+
parameters=ApplicationStatsAPI.get_parameters(),
60+
responses=ApplicationStatsAPI.get_response(),
61+
tags=[_('Application')] # type: ignore
62+
)
63+
@has_permissions(PermissionConstants.APPLICATION_OVERVIEW_READ.get_workspace_application_permission(),
64+
PermissionConstants.APPLICATION_OVERVIEW_READ.get_workspace_permission_workspace_manage_role(),
65+
ViewPermission([RoleConstants.USER.get_workspace_role()],
66+
[PermissionConstants.APPLICATION.get_workspace_application_permission()],
67+
CompareConstants.AND),
68+
RoleConstants.WORKSPACE_MANAGE.get_workspace_role())
69+
def get(self, request: Request, workspace_id: str, application_id: str):
70+
return result.success(
71+
ApplicationStatisticsSerializer(data={'application_id': application_id, 'workspace_id': workspace_id,
72+
'start_time': request.query_params.get(
73+
'start_time'),
74+
'end_time': request.query_params.get(
75+
'end_time')
76+
}).get_token_usage_statistics())
77+
78+
class TopQuestionsStatistics(APIView):
79+
authentication_classes = [TokenAuth]
80+
# 应用的top10问题统计
81+
@extend_schema(
82+
methods=['GET'],
83+
description=_('Application top10 question statistics'),
84+
summary=_('Application top10 question statistics'),
85+
operation_id=_('Application top10 question statistics'), # type: ignore
86+
parameters=ApplicationStatsAPI.get_parameters(),
87+
responses=ApplicationStatsAPI.get_response(),
88+
tags=[_('Application')] # type: ignore
89+
)
90+
@has_permissions(PermissionConstants.APPLICATION_OVERVIEW_READ.get_workspace_application_permission(),
91+
PermissionConstants.APPLICATION_OVERVIEW_READ.get_workspace_permission_workspace_manage_role(),
92+
ViewPermission([RoleConstants.USER.get_workspace_role()],
93+
[PermissionConstants.APPLICATION.get_workspace_application_permission()],
94+
CompareConstants.AND),
95+
RoleConstants.WORKSPACE_MANAGE.get_workspace_role())
96+
def get(self, request: Request, workspace_id: str, application_id: str):
97+
return result.success(
98+
ApplicationStatisticsSerializer(data={'application_id': application_id, 'workspace_id': workspace_id,
99+
'start_time': request.query_params.get(
100+
'start_time'),
101+
'end_time': request.query_params.get(
102+
'end_time')
103+
}).get_top_questions_statistics())

‎apps/models_provider/impl/base_chat_open_ai.py‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
# coding=utf-8
2+
import base64
23
from concurrent.futures import ThreadPoolExecutor
34
from requests.exceptions import ConnectTimeout, ReadTimeout
45
from typing import Dict, Optional, Any, Iterator, cast, Union, Sequence, Callable, Mapping
@@ -211,3 +212,20 @@ def invoke(
211212
self.usage_metadata = chat_result.response_metadata[
212213
'token_usage'] if 'token_usage' in chat_result.response_metadata else chat_result.usage_metadata
213214
return chat_result
215+
216+
217+
def upload_file_and_get_url(self, file_stream, file_name):
218+
"""上传文件并获取文件URL"""
219+
base64_video = base64.b64encode(file_stream).decode("utf-8")
220+
video_format = get_video_format(file_name)
221+
return f'data:{video_format};base64,{base64_video}'
222+
223+
def get_video_format(file_name):
224+
extension = file_name.split('.')[-1].lower()
225+
format_map = {
226+
'mp4': 'video/mp4',
227+
'avi': 'video/avi',
228+
'mov': 'video/mov',
229+
'wmv': 'video/x-ms-wmv'
230+
}
231+
return format_map.get(extension, 'video/mp4')

‎apps/models_provider/impl/volcanic_engine_model_provider/model/image.py‎

Lines changed: 2 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -25,20 +25,5 @@ def new_instance(model_type, model_name, model_credential: Dict[str, object], **
2525
def is_cache_model():
2626
return False
2727

28-
def upload_file_and_get_url(self, file_stream, file_name):
29-
"""上传文件并获取文件URL"""
30-
base64_video = base64.b64encode(file_stream).decode("utf-8")
31-
video_format = get_video_format(file_name)
32-
return f'data:{video_format};base64,{base64_video}'
33-
34-
35-
36-
def get_video_format(file_name):
37-
extension = file_name.split('.')[-1].lower()
38-
format_map = {
39-
'mp4': 'video/mp4',
40-
'avi': 'video/avi',
41-
'mov': 'video/mov',
42-
'wmv': 'video/x-ms-wmv'
43-
}
44-
return format_map.get(extension, 'video/mp4')
28+
29+

‎apps/models_provider/impl/zhipu_model_provider/model/image.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@
33
from models_provider.base_model_provider import MaxKBBaseModel
44
from models_provider.impl.base_chat_open_ai import BaseChatOpenAI
55

6-
76
class ZhiPuImage(MaxKBBaseModel, BaseChatOpenAI):
87

98
@staticmethod

0 commit comments

Comments
 (0)