Skip to content

Commit e7070b4

Browse files
liuyunchao-1998liuyunchao-1998
authored
fix(knowledge): Fix the issue of ineffective pagination parameters in the image-based knowledge base (#831)
1 parent f956c18 commit e7070b4

File tree

6 files changed

+168
-59
lines changed

6 files changed

+168
-59
lines changed

‎backend/domain/knowledge/entity/slice.go‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,15 @@ type WhereSliceOpt struct {
2727
DocumentID int64
2828
DocumentIDs []int64
2929
Keyword *string
30-
Sequence int64
3130
PageSize int64
3231
Offset int64
3332
NotEmpty *bool
3433
}
34+
35+
type WherePhotoSliceOpt struct {
36+
KnowledgeID int64
37+
DocumentIDs []int64
38+
Limit *int
39+
Offset *int
40+
HasCaption *bool
41+
}

‎backend/domain/knowledge/internal/dal/dao/knowledge_document_slice.go‎

Lines changed: 45 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -236,8 +236,11 @@ func (dao *KnowledgeDocumentSliceDAO) FindSliceByCondition(ctx context.Context,
236236

237237
if opts.PageSize != 0 {
238238
do = do.Limit(int(opts.PageSize))
239-
do = do.Offset(int(opts.Sequence)).Order(s.Sequence.Asc())
240239
}
240+
if opts.Offset != 0 {
241+
do = do.Offset(int(opts.Offset))
242+
}
243+
do = do.Order(s.Sequence.Asc())
241244
if opts.NotEmpty != nil {
242245
if ptr.From(opts.NotEmpty) {
243246
do = do.Where(s.Content.Neq(""))
@@ -319,3 +322,44 @@ func (dao *KnowledgeDocumentSliceDAO) GetLastSequence(ctx context.Context, docum
319322
}
320323
return resp.Sequence, nil
321324
}
325+
326+
func (dao *KnowledgeDocumentSliceDAO) ListPhotoSlice(ctx context.Context, opts *entity.WherePhotoSliceOpt) ([]*model.KnowledgeDocumentSlice, int64, error) {
327+
s := dao.Query.KnowledgeDocumentSlice
328+
do := s.WithContext(ctx)
329+
if opts.KnowledgeID != 0 {
330+
do = do.Where(s.KnowledgeID.Eq(opts.KnowledgeID))
331+
}
332+
if len(opts.DocumentIDs) != 0 {
333+
do = do.Where(s.DocumentID.In(opts.DocumentIDs...))
334+
}
335+
if ptr.From(opts.Limit) != 0 {
336+
do = do.Limit(int(ptr.From(opts.Limit)))
337+
}
338+
if ptr.From(opts.Offset) != 0 {
339+
do = do.Offset(int(ptr.From(opts.Offset)))
340+
}
341+
if opts.HasCaption != nil {
342+
if ptr.From(opts.HasCaption) {
343+
do = do.Where(s.Content.Neq(""))
344+
} else {
345+
do = do.Where(s.Content.Eq(""))
346+
}
347+
}
348+
do = do.Order(s.UpdatedAt.Desc())
349+
pos, err := do.Find()
350+
if err != nil {
351+
return nil, 0, err
352+
}
353+
total, err := do.Limit(-1).Offset(-1).Count()
354+
if err != nil {
355+
return nil, 0, err
356+
}
357+
return pos, total, nil
358+
}
359+
360+
func (dao *KnowledgeDocumentSliceDAO) BatchCreateWithTX(ctx context.Context, tx *gorm.DB, slices []*model.KnowledgeDocumentSlice) error {
361+
if len(slices) == 0 {
362+
return nil
363+
}
364+
return tx.WithContext(ctx).Debug().Model(&model.KnowledgeDocumentSlice{}).CreateInBatches(slices, 100).Error
365+
}

‎backend/domain/knowledge/processor/impl/base.go‎

Lines changed: 31 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -49,8 +49,9 @@ type baseDocProcessor struct {
4949
documentSource *entity.DocumentSource
5050

5151
// Drop DB model
52-
TableName string
53-
docModels []*model.KnowledgeDocument
52+
TableName string
53+
docModels []*model.KnowledgeDocument
54+
imageSlices []*model.KnowledgeDocumentSlice
5455

5556
storage storage.Storage
5657
knowledgeRepo repository.KnowledgeRepo
@@ -69,14 +70,14 @@ func (p *baseDocProcessor) BeforeCreate() error {
6970

7071
func (p *baseDocProcessor) BuildDBModel() error {
7172
p.docModels = make([]*model.KnowledgeDocument, 0, len(p.Documents))
72-
ids, err := p.idgen.GenMultiIDs(p.ctx, len(p.Documents))
73-
if err != nil {
74-
logs.CtxErrorf(p.ctx, "gen ids failed, err: %v", err)
75-
return errorx.New(errno.ErrKnowledgeIDGenCode)
76-
}
7773
for i := range p.Documents {
74+
id, err := p.idgen.GenID(p.ctx)
75+
if err != nil {
76+
logs.CtxErrorf(p.ctx, "gen id failed, err: %v", err)
77+
return errorx.New(errno.ErrKnowledgeIDGenCode)
78+
}
7879
docModel := &model.KnowledgeDocument{
79-
ID: ids[i],
80+
ID: id,
8081
KnowledgeID: p.Documents[i].KnowledgeID,
8182
Name: p.Documents[i].Name,
8283
FileExtension: string(p.Documents[i].FileExtension),
@@ -95,6 +96,23 @@ func (p *baseDocProcessor) BuildDBModel() error {
9596
}
9697
p.Documents[i].ID = docModel.ID
9798
p.docModels = append(p.docModels, docModel)
99+
if p.Documents[i].Type == knowledge.DocumentTypeImage {
100+
id, err := p.idgen.GenID(p.ctx)
101+
if err != nil {
102+
logs.CtxErrorf(p.ctx, "gen id failed, err: %v", err)
103+
return errorx.New(errno.ErrKnowledgeIDGenCode)
104+
}
105+
p.imageSlices = append(p.imageSlices, &model.KnowledgeDocumentSlice{
106+
ID: id,
107+
KnowledgeID: p.Documents[i].KnowledgeID,
108+
DocumentID: p.Documents[i].ID,
109+
CreatedAt: time.Now().UnixMilli(),
110+
UpdatedAt: time.Now().UnixMilli(),
111+
CreatorID: p.UserID,
112+
SpaceID: p.SpaceID,
113+
Status: int32(knowledge.SliceStatusInit),
114+
})
115+
}
98116
}
99117

100118
return nil
@@ -142,6 +160,11 @@ func (p *baseDocProcessor) InsertDBModel() (err error) {
142160
logs.CtxErrorf(ctx, "create document failed, err: %v", err)
143161
return errorx.New(errno.ErrKnowledgeDBCode, errorx.KV("msg", err.Error()))
144162
}
163+
err = p.sliceRepo.BatchCreateWithTX(ctx, tx, p.imageSlices)
164+
if err != nil {
165+
logs.CtxErrorf(ctx, "update knowledge failed, err: %v", err)
166+
return errorx.New(errno.ErrKnowledgeDBCode, errorx.KV("msg", err.Error()))
167+
}
145168
err = p.knowledgeRepo.UpdateWithTx(ctx, tx, p.Documents[0].KnowledgeID, map[string]interface{}{
146169
"updated_at": time.Now().UnixMilli(),
147170
})

‎backend/domain/knowledge/repository/repository.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -84,12 +84,12 @@ type KnowledgeDocumentSliceRepo interface {
8484
Create(ctx context.Context, slice *model.KnowledgeDocumentSlice) error
8585
Update(ctx context.Context, slice *model.KnowledgeDocumentSlice) error
8686
Delete(ctx context.Context, slice *model.KnowledgeDocumentSlice) error
87-
87+
BatchCreateWithTX(ctx context.Context, tx *gorm.DB, slices []*model.KnowledgeDocumentSlice) error
8888
BatchCreate(ctx context.Context, slices []*model.KnowledgeDocumentSlice) error
8989
BatchSetStatus(ctx context.Context, ids []int64, status int32, reason string) error
9090
DeleteByDocument(ctx context.Context, documentID int64) error
9191
MGetSlices(ctx context.Context, sliceIDs []int64) ([]*model.KnowledgeDocumentSlice, error)
92-
92+
ListPhotoSlice(ctx context.Context, opts *entity.WherePhotoSliceOpt) ([]*model.KnowledgeDocumentSlice, int64, error)
9393
FindSliceByCondition(ctx context.Context, opts *entity.WhereSliceOpt) (
9494
[]*model.KnowledgeDocumentSlice, int64, error)
9595
GetDocumentSliceIDs(ctx context.Context, docIDs []int64) (sliceIDs []int64, err error)

‎backend/domain/knowledge/service/event_handle.go‎

Lines changed: 77 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -190,24 +190,26 @@ func (k *knowledgeSVC) indexDocument(ctx context.Context, event *entity.Event) (
190190
collectionName := getCollectionName(doc.KnowledgeID)
191191

192192
if !doc.IsAppend {
193-
ids, err := k.sliceRepo.GetDocumentSliceIDs(ctx, []int64{doc.ID})
194-
if err != nil {
195-
return errorx.New(errno.ErrKnowledgeDBCode, errorx.KV("msg", fmt.Sprintf("get document slice ids failed, err: %v", err)))
196-
}
197-
if len(ids) > 0 {
198-
if err = k.sliceRepo.DeleteByDocument(ctx, doc.ID); err != nil {
199-
return errorx.New(errno.ErrKnowledgeDBCode, errorx.KV("msg", fmt.Sprintf("delete document slice failed, err: %v", err)))
193+
if doc.Type != knowledge.DocumentTypeImage {
194+
ids, err := k.sliceRepo.GetDocumentSliceIDs(ctx, []int64{doc.ID})
195+
if err != nil {
196+
return errorx.New(errno.ErrKnowledgeDBCode, errorx.KV("msg", fmt.Sprintf("get document slice ids failed, err: %v", err)))
200197
}
201-
for _, manager := range k.searchStoreManagers {
202-
s, err := manager.GetSearchStore(ctx, collectionName)
203-
if err != nil {
204-
return errorx.New(errno.ErrKnowledgeSearchStoreCode, errorx.KV("msg", fmt.Sprintf("get search store failed, err: %v", err)))
198+
if len(ids) > 0 {
199+
if err = k.sliceRepo.DeleteByDocument(ctx, doc.ID); err != nil {
200+
return errorx.New(errno.ErrKnowledgeDBCode, errorx.KV("msg", fmt.Sprintf("delete document slice failed, err: %v", err)))
205201
}
206-
if err := s.Delete(ctx, slices.Transform(event.SliceIDs, func(id int64) string {
207-
return strconv.FormatInt(id, 10)
208-
})); err != nil {
209-
logs.Errorf("[indexDocument] delete knowledge failed, err: %v", err)
210-
return errorx.New(errno.ErrKnowledgeSearchStoreCode, errorx.KV("msg", fmt.Sprintf("delete search store failed, err: %v", err)))
202+
for _, manager := range k.searchStoreManagers {
203+
s, err := manager.GetSearchStore(ctx, collectionName)
204+
if err != nil {
205+
return errorx.New(errno.ErrKnowledgeSearchStoreCode, errorx.KV("msg", fmt.Sprintf("get search store failed, err: %v", err)))
206+
}
207+
if err := s.Delete(ctx, slices.Transform(event.SliceIDs, func(id int64) string {
208+
return strconv.FormatInt(id, 10)
209+
})); err != nil {
210+
logs.Errorf("[indexDocument] delete knowledge failed, err: %v", err)
211+
return errorx.New(errno.ErrKnowledgeSearchStoreCode, errorx.KV("msg", fmt.Sprintf("delete search store failed, err: %v", err)))
212+
}
211213
}
212214
}
213215
}
@@ -298,34 +300,68 @@ func (k *knowledgeSVC) indexDocument(ctx context.Context, event *entity.Event) (
298300
seqOffset += 1
299301
}
300302

301-
sliceModels := make([]*model.KnowledgeDocumentSlice, 0, len(parseResult))
302-
for i, src := range parseResult {
303-
now := time.Now().UnixMilli()
304-
sliceModel := &model.KnowledgeDocumentSlice{
305-
ID: allIDs[i],
306-
KnowledgeID: doc.KnowledgeID,
307-
DocumentID: doc.ID,
308-
Content: parseResult[i].Content,
309-
Sequence: seqOffset + float64(i),
310-
CreatedAt: now,
311-
UpdatedAt: now,
312-
CreatorID: doc.CreatorID,
313-
SpaceID: doc.SpaceID,
314-
Status: int32(model.SliceStatusProcessing),
315-
FailReason: "",
316-
}
317-
if doc.Type == knowledge.DocumentTypeTable {
318-
sliceEntity, err := convertFn(src, doc.KnowledgeID, doc.ID, doc.CreatorID)
303+
if doc.Type == knowledge.DocumentTypeImage {
304+
if len(parseResult) != 0 {
305+
slices, _, err := k.sliceRepo.FindSliceByCondition(ctx, &entity.WhereSliceOpt{DocumentID: doc.ID})
319306
if err != nil {
320-
logs.CtxErrorf(ctx, "[indexDocument] convert document failed, err: %v", err)
321-
return errorx.New(errno.ErrKnowledgeSystemCode, errorx.KV("msg", fmt.Sprintf("convert document failed, err: %v", err)))
307+
return errorx.New(errno.ErrKnowledgeDBCode, errorx.KV("msg", fmt.Sprintf("find slice failed, err: %v", err)))
308+
}
309+
var slice *model.KnowledgeDocumentSlice
310+
if len(slices) > 0 {
311+
slice = slices[0]
312+
slice.Content = parseResult[0].Content
313+
} else {
314+
id, err := k.idgen.GenID(ctx)
315+
if err != nil {
316+
return errorx.New(errno.ErrKnowledgeIDGenCode, errorx.KV("msg", fmt.Sprintf("GenID failed, err: %v", err)))
317+
}
318+
slice = &model.KnowledgeDocumentSlice{
319+
ID: id,
320+
KnowledgeID: doc.KnowledgeID,
321+
DocumentID: doc.ID,
322+
Content: parseResult[0].Content,
323+
CreatedAt: time.Now().UnixMilli(),
324+
UpdatedAt: time.Now().UnixMilli(),
325+
CreatorID: doc.CreatorID,
326+
SpaceID: doc.SpaceID,
327+
Status: int32(model.SliceStatusProcessing),
328+
FailReason: "",
329+
}
330+
}
331+
if err = k.sliceRepo.Update(ctx, slice); err != nil {
332+
return errorx.New(errno.ErrKnowledgeDBCode, errorx.KV("msg", fmt.Sprintf("update slice failed, err: %v", err)))
322333
}
323-
sliceModel.Content = sliceEntity.GetSliceContent()
324334
}
325-
sliceModels = append(sliceModels, sliceModel)
326-
}
327-
if err = k.sliceRepo.BatchCreate(ctx, sliceModels); err != nil {
328-
return errorx.New(errno.ErrKnowledgeDBCode, errorx.KV("msg", fmt.Sprintf("batch create slice failed, err: %v", err)))
335+
} else {
336+
sliceModels := make([]*model.KnowledgeDocumentSlice, 0, len(parseResult))
337+
for i, src := range parseResult {
338+
now := time.Now().UnixMilli()
339+
sliceModel := &model.KnowledgeDocumentSlice{
340+
ID: allIDs[i],
341+
KnowledgeID: doc.KnowledgeID,
342+
DocumentID: doc.ID,
343+
Content: parseResult[i].Content,
344+
Sequence: seqOffset + float64(i),
345+
CreatedAt: now,
346+
UpdatedAt: now,
347+
CreatorID: doc.CreatorID,
348+
SpaceID: doc.SpaceID,
349+
Status: int32(model.SliceStatusProcessing),
350+
FailReason: "",
351+
}
352+
if doc.Type == knowledge.DocumentTypeTable {
353+
sliceEntity, err := convertFn(src, doc.KnowledgeID, doc.ID, doc.CreatorID)
354+
if err != nil {
355+
logs.CtxErrorf(ctx, "[indexDocument] convert document failed, err: %v", err)
356+
return errorx.New(errno.ErrKnowledgeSystemCode, errorx.KV("msg", fmt.Sprintf("convert document failed, err: %v", err)))
357+
}
358+
sliceModel.Content = sliceEntity.GetSliceContent()
359+
}
360+
sliceModels = append(sliceModels, sliceModel)
361+
}
362+
if err = k.sliceRepo.BatchCreate(ctx, sliceModels); err != nil {
363+
return errorx.New(errno.ErrKnowledgeDBCode, errorx.KV("msg", fmt.Sprintf("batch create slice failed, err: %v", err)))
364+
}
329365
}
330366

331367
defer func() {

‎backend/domain/knowledge/service/knowledge.go‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -876,9 +876,8 @@ func (k *knowledgeSVC) ListSlice(ctx context.Context, request *ListSliceRequest)
876876
KnowledgeID: ptr.From(request.KnowledgeID),
877877
DocumentID: ptr.From(request.DocumentID),
878878
Keyword: request.Keyword,
879-
Sequence: request.Sequence,
879+
Offset: request.Sequence,
880880
PageSize: request.Limit,
881-
Offset: request.Offset,
882881
})
883882
if err != nil {
884883
logs.CtxErrorf(ctx, "list slice failed, err: %v", err)
@@ -1375,12 +1374,12 @@ func (k *knowledgeSVC) ListPhotoSlice(ctx context.Context, request *ListPhotoSli
13751374
if request == nil {
13761375
return nil, errorx.New(errno.ErrKnowledgeInvalidParamCode, errorx.KV("msg", "request is empty"))
13771376
}
1378-
sliceArr, total, err := k.sliceRepo.FindSliceByCondition(ctx, &entity.WhereSliceOpt{
1377+
sliceArr, total, err := k.sliceRepo.ListPhotoSlice(ctx, &entity.WherePhotoSliceOpt{
13791378
KnowledgeID: request.KnowledgeID,
13801379
DocumentIDs: request.DocumentIDs,
1381-
Offset: int64(ptr.From(request.Offset)),
1382-
PageSize: int64(ptr.From(request.Limit)),
1383-
NotEmpty: request.HasCaption,
1380+
Offset: request.Offset,
1381+
Limit: request.Limit,
1382+
HasCaption: request.HasCaption,
13841383
})
13851384
if err != nil {
13861385
return nil, errorx.New(errno.ErrKnowledgeDBCode, errorx.KV("msg", err.Error()))

0 commit comments

Comments
 (0)