Skip to content

Commit 3b5a023

Browse files
liuruibinliuruibin
committed
refactor: celery config
1 parent 870fabe commit 3b5a023

File tree

5 files changed

+283
-0
lines changed

5 files changed

+283
-0
lines changed

‎apps/knowledge/migrations/0002_document_paragraph_embedding_problem_and_more.py‎

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
# Generated by Django 5.2 on 2025-04-29 03:28
2+
3+
import django.contrib.postgres.search
4+
import django.db.models.deletion
5+
import knowledge.models.knowledge
6+
import uuid_utils.compat
7+
from django.db import migrations, models
8+
9+
10+
class Migration(migrations.Migration):
11+
12+
dependencies = [
13+
('knowledge', '0001_initial'),
14+
]
15+
16+
operations = [
17+
migrations.CreateModel(
18+
name='Document',
19+
fields=[
20+
('create_time', models.DateTimeField(auto_now_add=True, verbose_name='创建时间')),
21+
('update_time', models.DateTimeField(auto_now=True, verbose_name='修改时间')),
22+
('id', models.UUIDField(default=uuid_utils.compat.uuid7, editable=False, primary_key=True, serialize=False, verbose_name='主键id')),
23+
('name', models.CharField(max_length=150, verbose_name='文档名称')),
24+
('char_length', models.IntegerField(verbose_name='文档字符数 冗余字段')),
25+
('status', models.CharField(default=knowledge.models.knowledge.Status.__str__, max_length=20, verbose_name='状态')),
26+
('status_meta', models.JSONField(default=knowledge.models.knowledge.default_status_meta, verbose_name='状态统计数据')),
27+
('is_active', models.BooleanField(default=True)),
28+
('type', models.IntegerField(choices=[(0, '通用类型'), (1, 'web站点类型'), (2, '飞书类型'), (3, '语雀类型')], default=0, verbose_name='类型')),
29+
('hit_handling_method', models.CharField(choices=[('optimization', '模型优化'), ('directly_return', '直接返回')], default='optimization', max_length=20, verbose_name='命中处理方式')),
30+
('directly_return_similarity', models.FloatField(default=0.9, verbose_name='直接回答相似度')),
31+
('meta', models.JSONField(default=dict, verbose_name='元数据')),
32+
('knowledge', models.ForeignKey(on_delete=django.db.models.deletion.DO_NOTHING, to='knowledge.knowledge', verbose_name='知识库id')),
33+
],
34+
options={
35+
'db_table': 'document',
36+
},
37+
),
38+
migrations.CreateModel(
39+
name='Paragraph',
40+
fields=[
41+
('create_time', models.DateTimeField(auto_now_add=True, verbose_name='创建时间')),
42+
('update_time', models.DateTimeField(auto_now=True, verbose_name='修改时间')),
43+
('id', models.UUIDField(default=uuid_utils.compat.uuid7, editable=False, primary_key=True, serialize=False, verbose_name='主键id')),
44+
('content', models.CharField(max_length=102400, verbose_name='段落内容')),
45+
('title', models.CharField(default='', max_length=256, verbose_name='标题')),
46+
('status', models.CharField(default=knowledge.models.knowledge.Status.__str__, max_length=20, verbose_name='状态')),
47+
('status_meta', models.JSONField(default=knowledge.models.knowledge.default_status_meta, verbose_name='状态数据')),
48+
('hit_num', models.IntegerField(default=0, verbose_name='命中次数')),
49+
('is_active', models.BooleanField(default=True)),
50+
('document', models.ForeignKey(db_constraint=False, on_delete=django.db.models.deletion.DO_NOTHING, to='knowledge.document')),
51+
('knowledge', models.ForeignKey(on_delete=django.db.models.deletion.DO_NOTHING, to='knowledge.knowledge')),
52+
],
53+
options={
54+
'db_table': 'paragraph',
55+
},
56+
),
57+
migrations.CreateModel(
58+
name='Embedding',
59+
fields=[
60+
('id', models.CharField(max_length=128, primary_key=True, serialize=False, verbose_name='主键id')),
61+
('source_id', models.CharField(max_length=128, verbose_name='资源id')),
62+
('source_type', models.CharField(choices=[(0, '问题'), (1, '段落'), (2, '标题')], default=0, max_length=5, verbose_name='资源类型')),
63+
('is_active', models.BooleanField(default=True, max_length=1, verbose_name='是否可用')),
64+
('embedding', knowledge.models.knowledge.VectorField(verbose_name='向量')),
65+
('search_vector', django.contrib.postgres.search.SearchVectorField(default='', verbose_name='分词')),
66+
('meta', models.JSONField(default=dict, verbose_name='元数据')),
67+
('document', models.ForeignKey(db_constraint=False, on_delete=django.db.models.deletion.DO_NOTHING, to='knowledge.document', verbose_name='文档关联')),
68+
('knowledge', models.ForeignKey(db_constraint=False, on_delete=django.db.models.deletion.DO_NOTHING, to='knowledge.knowledge', verbose_name='文档关联')),
69+
('paragraph', models.ForeignKey(db_constraint=False, on_delete=django.db.models.deletion.DO_NOTHING, to='knowledge.paragraph', verbose_name='段落关联')),
70+
],
71+
options={
72+
'db_table': 'embedding',
73+
},
74+
),
75+
migrations.CreateModel(
76+
name='Problem',
77+
fields=[
78+
('create_time', models.DateTimeField(auto_now_add=True, verbose_name='创建时间')),
79+
('update_time', models.DateTimeField(auto_now=True, verbose_name='修改时间')),
80+
('id', models.UUIDField(default=uuid_utils.compat.uuid7, editable=False, primary_key=True, serialize=False, verbose_name='主键id')),
81+
('content', models.CharField(max_length=256, verbose_name='问题内容')),
82+
('hit_num', models.IntegerField(default=0, verbose_name='命中次数')),
83+
('knowledge', models.ForeignKey(db_constraint=False, on_delete=django.db.models.deletion.DO_NOTHING, to='knowledge.knowledge')),
84+
],
85+
options={
86+
'db_table': 'problem',
87+
},
88+
),
89+
migrations.CreateModel(
90+
name='ProblemParagraphMapping',
91+
fields=[
92+
('create_time', models.DateTimeField(auto_now_add=True, verbose_name='创建时间')),
93+
('update_time', models.DateTimeField(auto_now=True, verbose_name='修改时间')),
94+
('id', models.UUIDField(default=uuid_utils.compat.uuid7, editable=False, primary_key=True, serialize=False, verbose_name='主键id')),
95+
('document', models.ForeignKey(on_delete=django.db.models.deletion.DO_NOTHING, to='knowledge.document')),
96+
('knowledge', models.ForeignKey(db_constraint=False, on_delete=django.db.models.deletion.DO_NOTHING, to='knowledge.knowledge')),
97+
('paragraph', models.ForeignKey(db_constraint=False, on_delete=django.db.models.deletion.DO_NOTHING, to='knowledge.paragraph')),
98+
('problem', models.ForeignKey(db_constraint=False, on_delete=django.db.models.deletion.DO_NOTHING, to='knowledge.problem')),
99+
],
100+
options={
101+
'db_table': 'problem_paragraph_mapping',
102+
},
103+
),
104+
]

‎apps/maxkb/settings/__init__.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,4 +7,6 @@
77
@desc:
88
"""
99
from .base import *
10+
from .logging import *
1011
from .auth import *
12+
from .lib import *

‎apps/maxkb/settings/base.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@
4444
'common',
4545
'system_manage',
4646
'models_provider',
47+
'django_celery_beat'
4748
]
4849

4950
MIDDLEWARE = [

‎apps/maxkb/settings/lib.py‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
# coding=utf-8
2+
"""
3+
@project: MaxKB
4+
@Author：虎
5+
@file： lib.py
6+
@date：2024/8/16 17:12
7+
@desc:
8+
"""
9+
import os
10+
import shutil
11+
12+
from maxkb.const import CONFIG, PROJECT_DIR
13+
14+
# celery相关配置
15+
celery_data_dir = os.path.join(PROJECT_DIR, 'data', 'celery_task')
16+
if not os.path.exists(celery_data_dir) or not os.path.isdir(celery_data_dir):
17+
os.makedirs(celery_data_dir)
18+
broker_path = os.path.join(celery_data_dir, "celery_db.sqlite3")
19+
backend_path = os.path.join(celery_data_dir, "celery_results.sqlite3")
20+
# 使用sql_lite 当做broker 和 响应接收
21+
CELERY_BROKER_URL = f'sqla+sqlite:///{broker_path}'
22+
CELERY_result_backend = f'db+sqlite:///{backend_path}'
23+
CELERY_timezone = CONFIG.TIME_ZONE
24+
CELERY_ENABLE_UTC = False
25+
CELERY_task_serializer = 'pickle'
26+
CELERY_result_serializer = 'pickle'
27+
CELERY_accept_content = ['json', 'pickle']
28+
CELERY_RESULT_EXPIRES = 600
29+
CELERY_WORKER_TASK_LOG_FORMAT = '%(asctime).19s %(message)s'
30+
CELERY_WORKER_LOG_FORMAT = '%(asctime).19s %(message)s'
31+
CELERY_TASK_EAGER_PROPAGATES = True
32+
CELERY_WORKER_REDIRECT_STDOUTS = True
33+
CELERY_WORKER_REDIRECT_STDOUTS_LEVEL = "INFO"
34+
CELERY_TASK_SOFT_TIME_LIMIT = 3600
35+
CELERY_WORKER_CANCEL_LONG_RUNNING_TASKS_ON_CONNECTION_LOSS = True
36+
CELERY_ACKS_LATE = True
37+
celery_once_path = os.path.join(celery_data_dir, "celery_once")
38+
try:
39+
if os.path.exists(celery_once_path) and os.path.isdir(celery_once_path):
40+
shutil.rmtree(celery_once_path)
41+
except Exception as e:
42+
pass
43+
CELERY_ONCE = {
44+
'backend': 'celery_once.backends.File',
45+
'settings': {'location': celery_once_path}
46+
}
47+
CELERY_BROKER_CONNECTION_RETRY_ON_STARTUP = True
48+
CELERY_LOG_DIR = os.path.join(PROJECT_DIR, 'logs', 'celery')

‎apps/maxkb/settings/logging.py‎

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
# -*- coding: utf-8 -*-
2+
#
3+
import os
4+
5+
from ..const import PROJECT_DIR, CONFIG
6+
7+
LOG_DIR = os.path.join(PROJECT_DIR, 'data', 'logs')
8+
MAX_KB_LOG_FILE = os.path.join(LOG_DIR, 'max_kb.log')
9+
DRF_EXCEPTION_LOG_FILE = os.path.join(LOG_DIR, 'drf_exception.log')
10+
UNEXPECTED_EXCEPTION_LOG_FILE = os.path.join(LOG_DIR, 'unexpected_exception.log')
11+
LOG_LEVEL = "DEBUG"
12+
13+
LOGGING = {
14+
'version': 1,
15+
'disable_existing_loggers': False,
16+
'formatters': {
17+
'verbose': {
18+
'format': '%(levelname)s %(asctime)s %(module)s %(process)d %(thread)d %(message)s'
19+
},
20+
'main': {
21+
'datefmt': '%Y-%m-%d %H:%M:%S',
22+
'format': '%(asctime)s [%(module)s %(levelname)s] %(message)s',
23+
},
24+
'exception': {
25+
'datefmt': '%Y-%m-%d %H:%M:%S',
26+
'format': '\n%(asctime)s [%(levelname)s] %(message)s',
27+
},
28+
'simple': {
29+
'format': '%(levelname)s %(message)s'
30+
},
31+
'syslog': {
32+
'format': 'jumpserver: %(message)s'
33+
},
34+
'msg': {
35+
'format': '%(message)s'
36+
}
37+
},
38+
'handlers': {
39+
'null': {
40+
'level': 'DEBUG',
41+
'class': 'logging.NullHandler',
42+
},
43+
'console': {
44+
'level': 'DEBUG',
45+
'class': 'logging.StreamHandler',
46+
'formatter': 'main'
47+
},
48+
'file': {
49+
'encoding': 'utf8',
50+
'level': 'DEBUG',
51+
'class': 'logging.handlers.RotatingFileHandler',
52+
'maxBytes': 1024 * 1024 * 100,
53+
'backupCount': 7,
54+
'formatter': 'main',
55+
'filename': MAX_KB_LOG_FILE,
56+
},
57+
'drf_exception': {
58+
'encoding': 'utf8',
59+
'level': 'DEBUG',
60+
'class': 'logging.handlers.RotatingFileHandler',
61+
'formatter': 'exception',
62+
'maxBytes': 1024 * 1024 * 100,
63+
'backupCount': 7,
64+
'filename': DRF_EXCEPTION_LOG_FILE,
65+
},
66+
'unexpected_exception': {
67+
'encoding': 'utf8',
68+
'level': 'DEBUG',
69+
'class': 'logging.handlers.RotatingFileHandler',
70+
'formatter': 'exception',
71+
'maxBytes': 1024 * 1024 * 100,
72+
'backupCount': 7,
73+
'filename': UNEXPECTED_EXCEPTION_LOG_FILE,
74+
},
75+
'syslog': {
76+
'level': 'INFO',
77+
'class': 'logging.NullHandler',
78+
'formatter': 'syslog'
79+
},
80+
},
81+
'loggers': {
82+
'django': {
83+
'handlers': ['null'],
84+
'propagate': False,
85+
'level': LOG_LEVEL,
86+
},
87+
'django.request': {
88+
'handlers': ['console', 'file', 'syslog'],
89+
'level': LOG_LEVEL,
90+
'propagate': False,
91+
},
92+
'sqlalchemy': {
93+
'handlers': ['console', 'file', 'syslog'],
94+
'level': "ERROR",
95+
'propagate': False,
96+
},
97+
'django.db.backends': {
98+
'handlers': ['console', 'file', 'syslog'],
99+
'propagate': False,
100+
'level': LOG_LEVEL,
101+
},
102+
'django.server': {
103+
'handlers': ['console', 'file', 'syslog'],
104+
'level': LOG_LEVEL,
105+
'propagate': False,
106+
},
107+
'max_kb_error': {
108+
'handlers': ['console', 'unexpected_exception'],
109+
'level': LOG_LEVEL,
110+
'propagate': False,
111+
},
112+
'max_kb': {
113+
'handlers': ['console', 'file'],
114+
'level': LOG_LEVEL,
115+
'propagate': False,
116+
},
117+
'common.event': {
118+
'handlers': ['console', 'file'],
119+
'level': "DEBUG",
120+
'propagate': False,
121+
},
122+
}
123+
}
124+
125+
SYSLOG_ENABLE = CONFIG.SYSLOG_ENABLE
126+
127+
if not os.path.isdir(LOG_DIR):
128+
os.makedirs(LOG_DIR, mode=0o755)

0 commit comments

Comments
 (0)