Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -307,10 +307,9 @@ def execute_block(self, message_list: List[BaseMessage],
else:
reasoning_content = reasoning_result.get('reasoning_content') + reasoning_result_end.get(
'reasoning_content')
asker = manage.context.get('form_data', {}).get('asker', None)
post_response_handler.handler(chat_id, chat_record_id, paragraph_list, problem_text,
chat_id, manage, self, padding_problem_text,
reasoning_content=reasoning_content if reasoning_content_enable else '')
content, manage, self, padding_problem_text,
reasoning_content=reasoning_content)
add_access_num(client_id, client_type, manage.context.get('application_id'))
return manage.get_base_to_response().to_block_response(str(chat_id), str(chat_record_id),
content, True,
Expand All @@ -324,10 +323,8 @@ def execute_block(self, message_list: List[BaseMessage],
except Exception as e:
all_text = 'Exception:' + str(e)
write_context(self, manage, 0, 0, all_text)
asker = manage.context.get('form_data', {}).get('asker', None)
post_response_handler.handler(chat_id, chat_record_id, paragraph_list, problem_text,
chat_id, manage, self, padding_problem_text,
reasoning_content='')
all_text, manage, self, padding_problem_text, reasoning_content='')
add_access_num(client_id, client_type, manage.context.get('application_id'))
return manage.get_base_to_response().to_block_response(str(chat_id), str(chat_record_id), all_text, True, 0,
0, _status=status.HTTP_500_INTERNAL_SERVER_ERROR)
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ def file_id_to_base64(file_id: str):
file = QuerySet(File).filter(id=file_id).first()
file_bytes = file.get_bytes()
base64_image = base64.b64encode(file_bytes).decode("utf-8")
return [base64_image, what(None, file_bytes.tobytes())]
return [base64_image, what(None, file_bytes)]


class BaseImageUnderstandNode(IImageUnderstandNode):
Expand Down Expand Up @@ -173,7 +173,7 @@ def generate_message_list(self, image_model, system: str, prompt: str, history_m
file = QuerySet(File).filter(id=file_id).first()
image_bytes = file.get_bytes()
base64_image = base64.b64encode(image_bytes).decode("utf-8")
image_format = what(None, image_bytes.tobytes())
image_format = what(None, image_bytes)
images.append(
{'type': 'image_url', 'image_url': {'url': f'data:image/{image_format};base64,{base64_image}'}})
messages = [HumanMessage(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,12 +22,16 @@ def get_default_global_variable(input_field_list: List):


def get_global_variable(node):
body = node.workflow_manage.get_body()
history_chat_record = node.flow_params_serializer.data.get('history_chat_record', [])
history_context = [{'question': chat_record.problem_text, 'answer': chat_record.answer_text} for chat_record in
history_chat_record]
chat_id = node.flow_params_serializer.data.get('chat_id')
return {'time': datetime.now().strftime('%Y-%m-%d %H:%M:%S'), 'start_time': time.time(),
'history_context': history_context, 'chat_id': str(chat_id), **node.workflow_manage.form_data}
'history_context': history_context, 'chat_id': str(chat_id), **node.workflow_manage.form_data,
'chat_user_id': body.get('chat_user_id'),
'chat_user_type': body.get('chat_user_type'),
'chat_user': body.get('chat_user')}


class BaseStartStepNode(IStarNode):
Expand Down Expand Up @@ -64,6 +68,7 @@ def execute(self, question, **kwargs) -> NodeResult:
'document': self.workflow_manage.document_list,
'audio': self.workflow_manage.audio_list,
'other': self.workflow_manage.other_list,

}
return NodeResult(node_variable, workflow_variable)

Expand Down
4 changes: 2 additions & 2 deletions apps/application/flow/workflow_manage.py
Original file line number Diff line number Diff line change
Expand Up @@ -154,8 +154,8 @@ def init_fields(self):
if global_fields is not None:
for global_field in global_fields:
global_field_list.append({**global_field, 'node_id': node_id, 'node_name': node_name})
field_list.sort(key=lambda f: len(f.get('node_name')), reverse=True)
global_field_list.sort(key=lambda f: len(f.get('node_name')), reverse=True)
field_list.sort(key=lambda f: len(f.get('node_name') + f.get('value')), reverse=True)
global_field_list.sort(key=lambda f: len(f.get('node_name') + f.get('value')), reverse=True)
self.field_list = field_list
self.global_field_list = global_field_list

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
# Generated by Django 5.2.3 on 2025-07-07 02:59

from django.db import migrations, models


class Migration(migrations.Migration):

dependencies = [
('application', '0004_application_publish_time_applicationversion_and_more'),
]

operations = [
migrations.AddField(
model_name='chat',
name='chat_record_count',
field=models.IntegerField(default=0, verbose_name='对话次数'),
),
migrations.AddField(
model_name='chat',
name='mark_sum',
field=models.IntegerField(default=0, verbose_name='标记数量'),
),
migrations.AddField(
model_name='chat',
name='star_num',
field=models.IntegerField(default=0, verbose_name='点赞数量'),
),
migrations.AddField(
model_name='chat',
name='trample_num',
field=models.IntegerField(default=0, verbose_name='点踩数量'),
),
]
4 changes: 4 additions & 0 deletions apps/application/models/application_chat.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,10 @@ class Chat(AppModelMixin):
is_deleted = models.BooleanField(verbose_name="逻辑删除", default=False)
asker = models.JSONField(verbose_name="访问者", default=default_asker, encoder=SystemEncoder)
meta = models.JSONField(verbose_name="元数据", default=dict)
star_num = models.IntegerField(verbose_name="点赞数量", default=0)
trample_num = models.IntegerField(verbose_name="点踩数量", default=0)
chat_record_count = models.IntegerField(verbose_name="对话次数", default=0)
mark_sum = models.IntegerField(verbose_name="标记数量", default=0)

class Meta:
db_table = "application_chat"
Expand Down
46 changes: 33 additions & 13 deletions apps/application/serializers/application_chat.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,12 +22,12 @@
from openpyxl.cell.cell import ILLEGAL_CHARACTERS_RE
from rest_framework import serializers

from application.models import Chat, Application
from application.models import Chat, Application, ChatRecord
from common.db.search import get_dynamics_model, native_search, native_page_search
from common.exception.app_exception import AppApiException
from common.utils.common import get_file_content
from maxkb.conf import PROJECT_DIR
from maxkb.settings import TIME_ZONE
from maxkb.settings import TIME_ZONE, edition


class ApplicationChatResponseSerializers(serializers.Serializer):
Expand Down Expand Up @@ -120,20 +120,17 @@ def get_query_set(self, select_ids=None):
condition = base_condition & min_trample_query
else:
condition = base_condition
inner_queryset = QuerySet(Chat).filter(application_id=self.data.get("application_id"))
if 'abstract' in self.data and self.data.get('abstract') is not None:
inner_queryset = inner_queryset.filter(abstract__icontains=self.data.get('abstract'))

return {
'inner_queryset': inner_queryset,
'default_queryset': query_set.filter(condition).order_by("-application_chat.update_time")
}

def list(self, with_valid=True):
if with_valid:
self.is_valid(raise_exception=True)
return native_search(self.get_query_set(), select_string=get_file_content(
os.path.join(PROJECT_DIR, "apps", "application", 'sql', 'list_application_chat.sql')),
os.path.join(PROJECT_DIR, "apps", "application", 'sql', ('list_application_chat_ee.sql' if ['PE', 'EE'].__contains__(
edition) else 'list_application_chat.sql'))),
with_table_name=False)

@staticmethod
Expand All @@ -144,7 +141,7 @@ def paragraph_list_to_string(paragraph_list):

@staticmethod
def to_row(row: Dict):
details = row.get('details')
details = row.get('details') or {}
padding_problem_text = ' '.join(node.get("answer", "") for key, node in details.items() if
node.get("type") == 'question-node')
search_dataset_node_list = [(key, node) for key, node in details.items() if
Expand All @@ -161,26 +158,28 @@ def to_row(row: Dict):
'name') + ':\n' + ApplicationChatQuerySerializers.paragraph_list_to_string(node.get('paragraph_list',
[])) for
key, node in search_dataset_node_list])
improve_paragraph_list = row.get('improve_paragraph_list')
improve_paragraph_list = row.get('improve_paragraph_list') or []
vote_status_map = {'-1': '未投票', '0': '赞同', '1': '反对'}
return [str(row.get('chat_id')), row.get('abstract'), row.get('problem_text'), padding_problem_text,
row.get('answer_text'), vote_status_map.get(row.get('vote_status')), reference_paragraph_len,
reference_paragraph,
"\n".join([
f"{improve_paragraph_list[index].get('title')}\n{improve_paragraph_list[index].get('content')}"
for index in range(len(improve_paragraph_list))]),
row.get('message_tokens') + row.get('answer_tokens'), row.get('run_time'),
(row.get('message_tokens') or 0) + (row.get('answer_tokens') or 0), row.get('run_time'),
str(row.get('create_time').astimezone(pytz.timezone(TIME_ZONE)).strftime('%Y-%m-%d %H:%M:%S')
)]
if row.get('create_time') is not None else None)]

def export(self, data, with_valid=True):
if with_valid:
self.is_valid(raise_exception=True)
ApplicationChatRecordExportRequest(data=data).is_valid(raise_exception=True)

data_list = native_search(self.get_query_set(data.get('select_ids')),
select_string=get_file_content(
os.path.join(PROJECT_DIR, "apps", "application", 'sql',
'export_application_chat.sql')),
('export_application_chat_ee.sql' if ['PE', 'EE'].__contains__(
edition) else 'export_application_chat.sql'))),
with_table_name=False)

batch_size = 500
Expand Down Expand Up @@ -232,5 +231,26 @@ def page(self, current_page: int, page_size: int, with_valid=True):
if with_valid:
self.is_valid(raise_exception=True)
return native_page_search(current_page, page_size, self.get_query_set(), select_string=get_file_content(
os.path.join(PROJECT_DIR, "apps", "application", 'sql', 'list_application_chat.sql')),
os.path.join(PROJECT_DIR, "apps", "application", 'sql',
('list_application_chat_ee.sql' if ['PE', 'EE'].__contains__(
edition) else 'list_application_chat.sql'))),
with_table_name=False)


class ChatCountSerializer(serializers.Serializer):
chat_id = serializers.UUIDField(required=True, label=_("Conversation ID"))

def get_query_set(self):
return QuerySet(ChatRecord).filter(chat_id=self.data.get('chat_id'))

def update_chat(self):
self.is_valid(raise_exception=True)
count_chat_record = native_search(self.get_query_set(), get_file_content(
os.path.join(PROJECT_DIR, "apps", "application", 'sql', 'count_chat_record.sql')), with_search_one=True)
QuerySet(Chat).filter(id=self.data.get('chat_id')).update(star_num=count_chat_record.get('star_num', 0) or 0,
trample_num=count_chat_record.get('trample_num',
0) or 0,
chat_record_count=count_chat_record.get(
'chat_record_count', 0) or 0,
mark_sum=count_chat_record.get('mark_sum', 0) or 0)
return True
2 changes: 2 additions & 0 deletions apps/application/serializers/application_chat_record.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
from rest_framework.utils.formatting import lazy_format

from application.models import ChatRecord, ApplicationAccessToken, Application
from application.serializers.application_chat import ChatCountSerializer
from application.serializers.common import ChatInfo
from common.db.search import page_search
from common.exception.app_exception import AppApiException
Expand Down Expand Up @@ -359,6 +360,7 @@ def improve(self, instance: Dict, with_valid=True):
update_document_char_length(document_id)
# 添加标注
chat_record.save()
ChatCountSerializer(data={'chat_id': chat_id}).update_chat()
return ChatRecordSerializerModel(chat_record).data, paragraph.id, knowledge_id

class Operate(serializers.Serializer):
Expand Down
29 changes: 27 additions & 2 deletions apps/application/serializers/common.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,9 @@
from django.utils.translation import gettext_lazy as _

from application.chat_pipeline.step.chat_step.i_chat_step import PostResponseHandler
from application.models import Application, ChatRecord, Chat, ApplicationVersion
from application.models import Application, ChatRecord, Chat, ApplicationVersion, ChatUserType
from common.constants.cache_version import Cache_Version
from common.database_model_manage.database_model_manage import DatabaseModelManage
from common.exception.app_exception import ChatException
from models_provider.models import Model
from models_provider.tools import get_model_credential
Expand Down Expand Up @@ -47,6 +48,7 @@ def __init__(self,
self.application_id = application_id
self.chat_record_list: List[ChatRecord] = []
self.application = None
self.chat_user = None
self.debug = debug

@staticmethod
Expand All @@ -73,8 +75,30 @@ def get_application(self):
self.application = application
return application

def get_chat_user(self, asker=None):
if self.chat_user:
return self.chat_user
if self.chat_user_type == ChatUserType.CHAT_USER.value:
chat_user_model = DatabaseModelManage.get_model("chat_user")
chat_user = QuerySet(chat_user_model).filter(id=self.chat_user_id).first()
return {
'id': chat_user.id,
'email': chat_user.email,
'phone': chat_user.phone,
'nick_name': chat_user.nick_name,
'username': chat_user.username,
'source': chat_user.source
}
else:
if asker:
self.chat_user = asker
else:
self.chat_user = {'username': '游客'}
return self.chat_user

def to_base_pipeline_manage_params(self):
self.get_application()
self.get_chat_user()
knowledge_setting = self.application.knowledge_setting
model_setting = self.application.model_setting
model_id = self.application.model_id
Expand Down Expand Up @@ -139,7 +163,8 @@ def append_chat_record(self, chat_record: ChatRecord):
if not self.debug:
if not QuerySet(Chat).filter(id=self.chat_id).exists():
Chat(id=self.chat_id, application_id=self.application_id, abstract=chat_record.problem_text[0:1024],
chat_user_id=self.chat_user_id, chat_user_type=self.chat_user_type).save()
chat_user_id=self.chat_user_id, chat_user_type=self.chat_user_type,
asker=self.get_chat_user()).save()
else:
QuerySet(Chat).filter(id=self.chat_id).update(update_time=datetime.now())
# 插入会话记录
Expand Down
7 changes: 7 additions & 0 deletions apps/application/sql/count_chat_record.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
SELECT COUNT
( "id" ) AS chat_record_count,
SUM ( CASE WHEN "vote_status" = '0' THEN 1 ELSE 0 END ) AS star_num,
SUM ( CASE WHEN "vote_status" = '1' THEN 1 ELSE 0 END ) AS trample_num,
SUM ( CASE WHEN array_length( application_chat_record.improve_paragraph_id_list, 1 ) IS NULL THEN 0 ELSE array_length( application_chat_record.improve_paragraph_id_list, 1 ) END ) AS mark_sum
FROM
application_chat_record
17 changes: 2 additions & 15 deletions apps/application/sql/export_application_chat.sql
Original file line number Diff line number Diff line change
Expand Up @@ -10,24 +10,11 @@ SELECT
application_chat_record_temp."index" as "index",
application_chat_record_temp.improve_paragraph_list as improve_paragraph_list,
application_chat_record_temp.vote_status as vote_status,
application_chat_record_temp.create_time as create_time
application_chat_record_temp.create_time as create_time,
application_chat.asker::json AS asker
FROM
application_chat application_chat
LEFT JOIN (
SELECT COUNT
( "id" ) AS chat_record_count,
SUM ( CASE WHEN "vote_status" = '0' THEN 1 ELSE 0 END ) AS star_num,
SUM ( CASE WHEN "vote_status" = '1' THEN 1 ELSE 0 END ) AS trample_num,
SUM ( CASE WHEN array_length( application_chat_record.improve_paragraph_id_list, 1 ) IS NULL THEN 0 ELSE array_length( application_chat_record.improve_paragraph_id_list, 1 ) END ) AS mark_sum,
chat_id
FROM
application_chat_record
WHERE chat_id IN (
SELECT id FROM application_chat ${inner_queryset})
GROUP BY
application_chat_record.chat_id
) chat_record_temp ON application_chat."id" = chat_record_temp.chat_id
LEFT JOIN (
SELECT
*,
CASE
Expand Down
28 changes: 28 additions & 0 deletions apps/application/sql/export_application_chat_ee.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
SELECT
application_chat."id" as chat_id,
application_chat.abstract as abstract,
application_chat_record_temp.problem_text as problem_text,
application_chat_record_temp.answer_text as answer_text,
application_chat_record_temp.message_tokens as message_tokens,
application_chat_record_temp.answer_tokens as answer_tokens,
application_chat_record_temp.run_time as run_time,
application_chat_record_temp.details::JSON as details,
application_chat_record_temp."index" as "index",
application_chat_record_temp.improve_paragraph_list as improve_paragraph_list,
application_chat_record_temp.vote_status as vote_status,
application_chat_record_temp.create_time as create_time,
(CASE WHEN "chat_user".id is NULL THEN application_chat.asker ELSE jsonb_build_object('id',chat_user.id,'user_name',chat_user.username) END)::json AS asker
FROM
application_chat application_chat
left join chat_user chat_user on chat_user.id::varchar = application_chat.chat_user_id
LEFT JOIN (
SELECT
*,
CASE
WHEN array_length( application_chat_record.improve_paragraph_id_list, 1 ) IS NULL THEN
'{}' ELSE ( SELECT ARRAY_AGG ( row_to_json ( paragraph ) ) FROM paragraph WHERE "id" = ANY ( application_chat_record.improve_paragraph_id_list ) )
END as improve_paragraph_list
FROM
application_chat_record application_chat_record
) application_chat_record_temp ON application_chat_record_temp.chat_id = application_chat."id"
${default_queryset}
Loading
Loading