From 71656a1f80f308e55a35d2259990954e7e5429dd Mon Sep 17 00:00:00 2001 From: cengsin Date: Wed, 24 Dec 2025 20:37:35 +0800 Subject: [PATCH 1/2] =?UTF-8?q?feat(retriever):=20=E6=B7=BB=E5=8A=A0?= =?UTF-8?q?=E8=BF=94=E5=9B=9E=E5=AD=97=E6=AE=B5=E5=92=8C=E6=96=87=E6=A1=A3?= =?UTF-8?q?=E8=BD=AC=E6=8D=A2=E5=99=A8=E9=85=8D=E7=BD=AE=E6=94=AF=E6=8C=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- components/retriever/qdrant/retriever.go | 89 ++++++++++++++++++------ 1 file changed, 68 insertions(+), 21 deletions(-) diff --git a/components/retriever/qdrant/retriever.go b/components/retriever/qdrant/retriever.go index ca4110091..80eff43b8 100644 --- a/components/retriever/qdrant/retriever.go +++ b/components/retriever/qdrant/retriever.go @@ -40,14 +40,20 @@ type Config struct { ScoreThreshold *float64 // Number of top results to retrieve from Qdrant. TopK int + // ReturnFields limits the attributes returned from the document's payload. num is the number of attributes following the keyword. + // Default []string{"content", "metadata"} + ReturnFields []string + // DocumentConverter converts retrieved raw document to eino Document, default defaultResultParser. + DocumentConverter func(ctx context.Context, point *qdrant.ScoredPoint) (*schema.Document, error) } type Retriever struct { - client *qdrant.Client - collection string - embedding embedding.Embedder - scoreThreshold *float64 - topK int + client *qdrant.Client + collection string + embedding embedding.Embedder + scoreThreshold *float64 + topK int + documentConverter func(ctx context.Context, point *qdrant.ScoredPoint) (*schema.Document, error) } func NewRetriever(ctx context.Context, config *Config) (*Retriever, error) { @@ -69,12 +75,21 @@ func NewRetriever(ctx context.Context, config *Config) (*Retriever, error) { topK = 5 } + if len(config.ReturnFields) == 0 { + config.ReturnFields = []string{defaultMetadataKey, defaultContentKey} + } + + if config.DocumentConverter == nil { + config.DocumentConverter = defaultResultParser(config.ReturnFields) + } + return &Retriever{ - client: config.Client, - collection: config.Collection, - embedding: config.Embedding, - scoreThreshold: config.ScoreThreshold, - topK: topK, + client: config.Client, + collection: config.Collection, + embedding: config.Embedding, + scoreThreshold: config.ScoreThreshold, + topK: topK, + documentConverter: config.DocumentConverter, }, nil } @@ -134,19 +149,13 @@ func (r *Retriever) Retrieve(ctx context.Context, query string, opts ...retrieve } docs = make([]*schema.Document, 0, len(resp)) for _, pt := range resp { - doc := &schema.Document{ - ID: pt.Id.GetUuid(), - MetaData: map[string]any{}, - } - - if val, ok := pt.Payload[defaultContentKey]; ok { - doc.Content = val.GetStringValue() + doc, err := r.documentConverter(ctx, pt) + if err != nil { + return nil, err } - - if val, ok := pt.Payload[defaultMetadataKey]; ok { - doc.MetaData[defaultMetadataKey] = val.GetStructValue().Fields + if doc == nil { + return nil, fmt.Errorf("[qdrant retriever] document converter returned nil document") } - doc.WithScore(float64(pt.Score)) docs = append(docs, doc) @@ -171,6 +180,44 @@ func (r *Retriever) makeEmbeddingCtx(ctx context.Context, emb embedding.Embedder return callbacks.ReuseHandlers(ctx, runInfo) } +func defaultResultParser(returnFields []string) func(ctx context.Context, point *qdrant.ScoredPoint) (*schema.Document, error) { + return func(ctx context.Context, point *qdrant.ScoredPoint) (*schema.Document, error) { + if point == nil { + return nil, fmt.Errorf("[defaultResultParser] point is nil") + } + + resp := &schema.Document{ + Content: "", + MetaData: map[string]any{}, + } + + if point.Id != nil { + if uuid := point.Id.GetUuid(); uuid != "" { + resp.ID = uuid + } else { + resp.ID = fmt.Sprintf("%d", point.Id.GetNum()) + } + } + + for _, field := range returnFields { + val, found := point.Payload[field] + if !found { + return nil, fmt.Errorf("[defaultResultParser] field=%s not found in payload, point=%v", field, point) + } + + if field == defaultContentKey { + resp.Content = val.GetStringValue() + } else if field == defaultMetadataKey { + resp.MetaData[defaultMetadataKey] = val.GetStructValue().Fields + } else { + resp.MetaData[field] = val + } + } + + return resp, nil + } +} + func tryMarshalJsonString(input any) string { if input == nil { return "" From 9dca857dc86d1ef755d5bbaf1193b01b3c230813 Mon Sep 17 00:00:00 2001 From: cengsin Date: Mon, 29 Dec 2025 17:03:57 +0800 Subject: [PATCH 2/2] =?UTF-8?q?feat(retriever):=20=E6=B7=BB=E5=8A=A0val?= =?UTF-8?q?=E8=A7=A3=E6=9E=90=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- components/retriever/qdrant/retriever.go | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/components/retriever/qdrant/retriever.go b/components/retriever/qdrant/retriever.go index 80eff43b8..8f3dcf7ca 100644 --- a/components/retriever/qdrant/retriever.go +++ b/components/retriever/qdrant/retriever.go @@ -210,7 +210,22 @@ func defaultResultParser(returnFields []string) func(ctx context.Context, point } else if field == defaultMetadataKey { resp.MetaData[defaultMetadataKey] = val.GetStructValue().Fields } else { - resp.MetaData[field] = val + switch val.GetKind().(type) { + case *qdrant.Value_NullValue: + resp.MetaData[field] = val.GetNullValue() + case *qdrant.Value_DoubleValue: + resp.MetaData[field] = val.GetDoubleValue() + case *qdrant.Value_IntegerValue: + resp.MetaData[field] = val.GetIntegerValue() + case *qdrant.Value_StringValue: + resp.MetaData[field] = val.GetStringValue() + case *qdrant.Value_BoolValue: + resp.MetaData[field] = val.GetBoolValue() + case *qdrant.Value_StructValue: + resp.MetaData[field] = val.GetStructValue().Fields + case *qdrant.Value_ListValue: + resp.MetaData[field] = val.GetListValue() + } } }