diff --git a/docs/mkdocs/en/knowledge.md b/docs/mkdocs/en/knowledge.md
index 9e91c2c4c..494bb0f77 100644
--- a/docs/mkdocs/en/knowledge.md
+++ b/docs/mkdocs/en/knowledge.md
@@ -183,21 +183,22 @@ kb := knowledge.New(
knowledge.WithEmbedder(embedder),
)
+// Note: Metadata fields must use the metadata. prefix
filterCondition := &searchfilter.UniversalFilterCondition{
Operator: searchfilter.OperatorAnd,
Value: []*searchfilter.UniversalFilterCondition{
{
- Field: "tag",
+ Field: "metadata.tag", // Metadata fields use metadata. prefix
Operator: searchfilter.OperatorEqual,
Value: "tag",
},
{
- Field: "age",
+ Field: "metadata.age",
Operator: searchfilter.OperatorGreaterThanOrEqual,
Value: 18,
},
{
- Field: "create_time",
+ Field: "metadata.create_time",
Operator: searchfilter.OperatorBetween,
Value: []string{"2024-10-11 12:11:00", "2025-10-11 12:11:00"},
},
@@ -205,12 +206,12 @@ filterCondition := &searchfilter.UniversalFilterCondition{
Operator: searchfilter.OperatorOr,
Value: []*searchfilter.UniversalFilterCondition{
{
- Field: "login_time",
+ Field: "metadata.login_time",
Operator: searchfilter.OperatorLessThanOrEqual,
Value: "2025-01-11 12:11:00",
},
{
- Field: "status",
+ Field: "metadata.status",
Operator: searchfilter.OperatorEqual,
Value: "logout",
},
@@ -828,9 +829,9 @@ llmAgent := llmagent.New(
"source": "official", // Official source
"category": "documentation", // Documentation category
}),
- // Agent-level complex condition filter
+ // Agent-level complex condition filter (metadata fields use metadata. prefix)
llmagent.WithKnowledgeConditionedFilter(
- searchfilter.Equal("status", "published"), // Published status
+ searchfilter.Equal("metadata.status", "published"), // Published status
),
)
@@ -844,12 +845,12 @@ eventCh, err := runner.Run(
}),
// Runner-level complex condition filter
agent.WithKnowledgeConditionedFilter(
- searchfilter.GreaterThan("priority", 5), // Priority greater than 5
+ searchfilter.GreaterThan("metadata.priority", 5), // Priority greater than 5
),
)
// 3. LLM intelligent filter (dynamically generated by LLM)
-// Example: User asks "find API related docs", LLM might generate {"topic": "api"}
+// Example: User asks "find API related docs", LLM might generate {"field": "metadata.topic", "value": "api"}
// Final effective filter conditions (all combined with AND):
// source = "official" AND
@@ -873,11 +874,11 @@ searchTool := tool.NewKnowledgeSearchTool(
tool.WithFilter(map[string]interface{}{
"source": "official",
}),
- // Agent-level complex condition filter
+ // Agent-level complex condition filter (metadata fields use metadata. prefix)
tool.WithConditionedFilter(
searchfilter.Or(
- searchfilter.Equal("topic", "programming"),
- searchfilter.Equal("topic", "llm"),
+ searchfilter.Equal("metadata.topic", "programming"),
+ searchfilter.Equal("metadata.topic", "llm"),
),
),
)
@@ -889,35 +890,57 @@ llmAgent := llmagent.New(
)
// Final filter conditions:
-// source = "official" AND (topic = "programming" OR topic = "llm")
+// metadata.source = "official" AND (metadata.topic = "programming" OR metadata.topic = "llm")
// i.e., must be official source AND topic is either programming or LLM
```
+##### Filter Field Naming Convention
+
+When using `FilterCondition`, **metadata fields must use the `metadata.` prefix**:
+
+```go
+// ✅ Correct: Use metadata. prefix
+searchfilter.Equal("metadata.topic", "programming")
+searchfilter.Equal("metadata.category", "documentation")
+
+// ❌ Wrong: Missing metadata. prefix
+searchfilter.Equal("topic", "programming")
+```
+
+> **Notes**:
+> - The `metadata.` prefix distinguishes metadata fields from system fields (e.g., `id`, `name`, `content`)
+> - If you customized the metadata field name via `WithMetadataField()`, still use the `metadata.` prefix; the framework will automatically convert it to the actual field name
+> - System fields (`id`, `name`, `content`, `created_at`, `updated_at`) use the field name directly without prefix
+
##### Common Filter Helper Functions
```go
-// Comparison operators
-searchfilter.Equal(field, value) // field = value
-searchfilter.NotEqual(field, value) // field != value
-searchfilter.GreaterThan(field, value) // field > value
-searchfilter.GreaterThanOrEqual(field, value) // field >= value
-searchfilter.LessThan(field, value) // field < value
-searchfilter.LessThanOrEqual(field, value) // field <= value
-searchfilter.In(field, values...) // field IN (...)
-searchfilter.NotIn(field, values...) // field NOT IN (...)
-searchfilter.Like(field, pattern) // field LIKE pattern
-searchfilter.Between(field, min, max) // field BETWEEN min AND max
+// Comparison operators (Note: metadata fields need metadata. prefix)
+searchfilter.Equal("metadata.topic", value) // metadata.topic = value
+searchfilter.NotEqual("metadata.status", value) // metadata.status != value
+searchfilter.GreaterThan("metadata.priority", value) // metadata.priority > value
+searchfilter.GreaterThanOrEqual("metadata.score", value) // metadata.score >= value
+searchfilter.LessThan("metadata.age", value) // metadata.age < value
+searchfilter.LessThanOrEqual("metadata.level", value) // metadata.level <= value
+searchfilter.In("metadata.category", values...) // metadata.category IN (...)
+searchfilter.NotIn("metadata.type", values...) // metadata.type NOT IN (...)
+searchfilter.Like("metadata.title", pattern) // metadata.title LIKE pattern
+searchfilter.Between("metadata.date", min, max) // metadata.date BETWEEN min AND max
+
+// System fields don't need prefix
+searchfilter.Equal("id", "doc-123") // id = "doc-123"
+searchfilter.In("name", "doc1", "doc2") // name IN ("doc1", "doc2")
// Logical operators
searchfilter.And(conditions...) // AND combination
searchfilter.Or(conditions...) // OR combination
-// Nested example: (status = 'published') AND (category = 'doc' OR category = 'tutorial')
+// Nested example: (metadata.status = 'published') AND (metadata.category = 'doc' OR metadata.category = 'tutorial')
searchfilter.And(
- searchfilter.Equal("status", "published"),
+ searchfilter.Equal("metadata.status", "published"),
searchfilter.Or(
- searchfilter.Equal("category", "documentation"),
- searchfilter.Equal("category", "tutorial"),
+ searchfilter.Equal("metadata.category", "documentation"),
+ searchfilter.Equal("metadata.category", "tutorial"),
),
)
```
diff --git a/docs/mkdocs/zh/knowledge.md b/docs/mkdocs/zh/knowledge.md
index aecf3cceb..099bc0736 100644
--- a/docs/mkdocs/zh/knowledge.md
+++ b/docs/mkdocs/zh/knowledge.md
@@ -180,21 +180,22 @@ kb := knowledge.New(
knowledge.WithEmbedder(embedder),
)
+// 注意：元数据字段需要使用 metadata. 前缀
filterCondition := &searchfilter.UniversalFilterCondition{
Operator: searchfilter.OperatorAnd,
Value: []*searchfilter.UniversalFilterCondition{
{
- Field: "tag",
+ Field: "metadata.tag", // 元数据字段使用 metadata. 前缀
Operator: searchfilter.OperatorEqual,
Value: "tag",
},
{
- Field: "age",
+ Field: "metadata.age",
Operator: searchfilter.OperatorGreaterThanOrEqual,
Value: 18,
},
{
- Field: "create_time",
+ Field: "metadata.create_time",
Operator: searchfilter.OperatorBetween,
Value: []string{"2024-10-11 12:11:00", "2025-10-11 12:11:00"},
},
@@ -202,12 +203,12 @@ filterCondition := &searchfilter.UniversalFilterCondition{
Operator: searchfilter.OperatorOr,
Value: []*searchfilter.UniversalFilterCondition{
{
- Field: "login_time",
+ Field: "metadata.login_time",
Operator: searchfilter.OperatorLessThanOrEqual,
Value: "2025-01-11 12:11:00",
},
{
- Field: "status",
+ Field: "metadata.status",
Operator: searchfilter.OperatorEqual,
Value: "logout",
},
@@ -826,9 +827,9 @@ llmAgent := llmagent.New(
"source": "official", // 官方来源
"category": "documentation", // 文档类别
}),
- // Agent 级复杂条件过滤器
+ // Agent 级复杂条件过滤器（元数据字段使用 metadata. 前缀）
llmagent.WithKnowledgeConditionedFilter(
- searchfilter.Equal("status", "published"), // 已发布状态
+ searchfilter.Equal("metadata.status", "published"), // 已发布状态
),
)
@@ -842,21 +843,21 @@ eventCh, err := runner.Run(
}),
// Runner 级复杂条件过滤器
agent.WithKnowledgeConditionedFilter(
- searchfilter.GreaterThan("priority", 5), // 优先级大于 5
+ searchfilter.GreaterThan("metadata.priority", 5), // 优先级大于 5
),
)
// 3. LLM 智能过滤器（由 LLM 动态生成）
-// 例如：用户问 "查找 API 相关文档"，LLM 可能生成 {"topic": "api"}
+// 例如：用户问 "查找 API 相关文档"，LLM 可能生成 {"field": "metadata.topic", "value": "api"}
// 最终生效的过滤条件（所有条件通过 AND 组合）：
-// source = "official" AND
-// category = "documentation" AND
-// status = "published" AND
-// region = "china" AND
-// language = "zh" AND
-// priority > 5 AND
-// topic = "api"
+// metadata.source = "official" AND
+// metadata.category = "documentation" AND
+// metadata.status = "published" AND
+// metadata.region = "china" AND
+// metadata.language = "zh" AND
+// metadata.priority > 5 AND
+// metadata.topic = "api"
//
// 即：必须同时满足所有层级的所有条件
```
@@ -871,11 +872,11 @@ searchTool := tool.NewKnowledgeSearchTool(
tool.WithFilter(map[string]any{
"source": "official",
}),
- // Agent 级复杂条件过滤器
+ // Agent 级复杂条件过滤器（元数据字段使用 metadata. 前缀）
tool.WithConditionedFilter(
searchfilter.Or(
- searchfilter.Equal("topic", "programming"),
- searchfilter.Equal("topic", "llm"),
+ searchfilter.Equal("metadata.topic", "programming"),
+ searchfilter.Equal("metadata.topic", "llm"),
),
),
)
@@ -887,35 +888,57 @@ llmAgent := llmagent.New(
)
// 最终过滤条件：
-// source = "official" AND (topic = "programming" OR topic = "llm")
+// metadata.source = "official" AND (metadata.topic = "programming" OR metadata.topic = "llm")
// 即：必须是官方来源，且主题是编程或 LLM
```
+##### 过滤器字段命名规范
+
+使用 `FilterCondition` 时，**元数据字段必须使用 `metadata.` 前缀**：
+
+```go
+// ✅ 正确：使用 metadata. 前缀
+searchfilter.Equal("metadata.topic", "programming")
+searchfilter.Equal("metadata.category", "documentation")
+
+// ❌ 错误：缺少 metadata. 前缀
+searchfilter.Equal("topic", "programming")
+```
+
+> **说明**：
+> - `metadata.` 前缀用于区分元数据字段和系统字段（如 `id`、`name`、`content` 等）
+> - 如果通过 `WithMetadataField()` 自定义了元数据字段名，仍然使用 `metadata.` 前缀，框架会自动转换为实际的字段名
+> - 系统字段（`id`、`name`、`content`、`created_at`、`updated_at`）直接使用字段名，无需前缀
+
##### 常用过滤器辅助函数
```go
-// 比较操作符
-searchfilter.Equal(field, value) // field = value
-searchfilter.NotEqual(field, value) // field != value
-searchfilter.GreaterThan(field, value) // field > value
-searchfilter.GreaterThanOrEqual(field, value) // field >= value
-searchfilter.LessThan(field, value) // field < value
-searchfilter.LessThanOrEqual(field, value) // field <= value
-searchfilter.In(field, values...) // field IN (...)
-searchfilter.NotIn(field, values...) // field NOT IN (...)
-searchfilter.Like(field, pattern) // field LIKE pattern
-searchfilter.Between(field, min, max) // field BETWEEN min AND max
+// 比较操作符（注意：元数据字段需要 metadata. 前缀）
+searchfilter.Equal("metadata.topic", value) // metadata.topic = value
+searchfilter.NotEqual("metadata.status", value) // metadata.status != value
+searchfilter.GreaterThan("metadata.priority", value) // metadata.priority > value
+searchfilter.GreaterThanOrEqual("metadata.score", value) // metadata.score >= value
+searchfilter.LessThan("metadata.age", value) // metadata.age < value
+searchfilter.LessThanOrEqual("metadata.level", value) // metadata.level <= value
+searchfilter.In("metadata.category", values...) // metadata.category IN (...)
+searchfilter.NotIn("metadata.type", values...) // metadata.type NOT IN (...)
+searchfilter.Like("metadata.title", pattern) // metadata.title LIKE pattern
+searchfilter.Between("metadata.date", min, max) // metadata.date BETWEEN min AND max
+
+// 系统字段不需要前缀
+searchfilter.Equal("id", "doc-123") // id = "doc-123"
+searchfilter.In("name", "doc1", "doc2") // name IN ("doc1", "doc2")
// 逻辑操作符
searchfilter.And(conditions...) // AND 组合
searchfilter.Or(conditions...) // OR 组合
-// 嵌套示例：(status = 'published') AND (category = 'doc' OR category = 'tutorial')
+// 嵌套示例：(metadata.status = 'published') AND (metadata.category = 'doc' OR metadata.category = 'tutorial')
searchfilter.And(
- searchfilter.Equal("status", "published"),
+ searchfilter.Equal("metadata.status", "published"),
searchfilter.Or(
- searchfilter.Equal("category", "documentation"),
- searchfilter.Equal("category", "tutorial"),
+ searchfilter.Equal("metadata.category", "documentation"),
+ searchfilter.Equal("metadata.category", "tutorial"),
),
)
```
diff --git a/examples/knowledge/.env b/examples/knowledge/.env
new file mode 100644
index 000000000..6b5560b0f
--- /dev/null
+++ b/examples/knowledge/.env
@@ -0,0 +1,3 @@
+export OPENAI_BASE_URL="http://wujipt.woa.com/api/v1"
+export OPENAI_API_KEY="2fe60304d5192f22"
+export MODEL_NAME="qwen3-omni-30b-a3b-thinking"
diff --git a/examples/knowledge/README.md b/examples/knowledge/README.md
new file mode 100644
index 000000000..005c37987
--- /dev/null
+++ b/examples/knowledge/README.md
@@ -0,0 +1,44 @@
+# Knowledge Examples
+
+Knowledge-enhanced AI agents examples.
+
+## Environment
+
+```bash
+export OPENAI_BASE_URL=xxx
+export OPENAI_API_KEY=xxx
+export MODEL_NAME=xxx
+```
+
+## Examples
+
+### basic/
+Basic example with file source and in-memory vector store.
+
+### sources/
+Different data source types:
+
+| Case | Description |
+|------|-------------|
+| file-source/ | Load individual files (Markdown, PDF, DOCX, CSV, JSON) |
+| directory-source/ | Recursively load entire directory |
+| url-source/ | Fetch and parse web pages |
+| auto-source/ | Auto-detect source type from path |
+
+### vectorstores/
+Persistent vector storage options:
+
+| Case | Description | Extra Environment |
+|------|-------------|-------------------|
+| postgres/ | PostgreSQL with pgvector extension | `PGVECTOR_HOST`, `PGVECTOR_PORT`, `PGVECTOR_USER`, `PGVECTOR_PASSWORD`, `PGVECTOR_DATABASE` |
+| elasticsearch/ | Elasticsearch (v7/v8/v9) | `ELASTICSEARCH_HOSTS`, `ELASTICSEARCH_USERNAME`, `ELASTICSEARCH_PASSWORD` |
+| tcvector/ | Tencent VectorDB | `TCVECTOR_URL`, `TCVECTOR_USERNAME`, `TCVECTOR_PASSWORD` |
+
+### features/
+Advanced features:
+
+| Case | Description |
+|------|-------------|
+| agentic-filter/ | LLM automatically generates metadata filter based on user query |
+| metadata-filter/ | Programmatic metadata filtering with AND/OR/NOT operations |
+| management/ | Dynamic source management: AddSource, RemoveSource, ReloadSource |
diff --git a/examples/knowledge/basic/README.md b/examples/knowledge/basic/README.md
new file mode 100644
index 000000000..aac0ff0cd
--- /dev/null
+++ b/examples/knowledge/basic/README.md
@@ -0,0 +1,47 @@
+# Basic Knowledge Example
+
+This is the simplest example to get started with knowledge-enhanced chat using trpc-agent-go.
+
+## What it demonstrates
+
+- Single file source
+- In-memory vector store
+- OpenAI embedder
+- Basic question answering
+- Command-line query option
+
+## Prerequisites
+
+Set your OpenAI API key:
+
+```bash
+export OPENAI_API_KEY=your-api-key
+export OPENAI_BASE_URL=https://api.openai.com/v1 # Optional
+export MODEL_NAME=deepseek-chat # Optional, defaults to deepseek-chat
+```
+
+## Run
+
+### With default query
+
+```bash
+go run main.go
+```
+
+### With custom query
+
+```bash
+go run main.go -query "What is a Large Language Model?"
+```
+
+## Example questions
+
+- "What is a Large Language Model?"
+- "How do LLMs work?"
+- "What are transformers?"
+
+## Next steps
+
+- **sources/**: Learn about different data sources (file, directory, URL, auto)
+- **vectorstores/**: Explore persistent storage options (PostgreSQL, Elasticsearch)
+- **features/**: Discover advanced features (agentic filter, streaming, metadata)
diff --git a/examples/knowledge/basic/main.go b/examples/knowledge/basic/main.go
new file mode 100644
index 000000000..6507f167a
--- /dev/null
+++ b/examples/knowledge/basic/main.go
@@ -0,0 +1,158 @@
+//
+// Tencent is pleased to support the open source community by making trpc-agent-go available.
+//
+// Copyright (C) 2025 Tencent. All rights reserved.
+//
+// trpc-agent-go is licensed under the Apache License Version 2.0.
+//
+//
+
+// Package main demonstrates basic knowledge integration.
+// This is the simplest example to get started with knowledge-enhanced chat.
+//
+// Required environment variables:
+// - OPENAI_API_KEY: Your OpenAI API key for LLM and embeddings
+// - OPENAI_BASE_URL: (Optional) Custom OpenAI API endpoint, defaults to https://api.openai.com/v1
+// - MODEL_NAME: (Optional) Model name to use, defaults to deepseek-chat
+//
+// Example usage:
+//
+// export OPENAI_API_KEY=sk-xxxx
+// export OPENAI_BASE_URL=https://api.openai.com/v1
+// export MODEL_NAME=deepseek-chat
+// go run main.go
+package main
+
+import (
+ "context"
+ "flag"
+ "fmt"
+ "log"
+ "os"
+ "strings"
+
+ "trpc.group/trpc-go/trpc-agent-go/agent/llmagent"
+ util "trpc.group/trpc-go/trpc-agent-go/examples/knowledge"
+ "trpc.group/trpc-go/trpc-agent-go/knowledge"
+ "trpc.group/trpc-go/trpc-agent-go/knowledge/embedder/openai"
+ "trpc.group/trpc-go/trpc-agent-go/knowledge/source"
+ "trpc.group/trpc-go/trpc-agent-go/knowledge/source/file"
+ knowledgetool "trpc.group/trpc-go/trpc-agent-go/knowledge/tool"
+ "trpc.group/trpc-go/trpc-agent-go/knowledge/vectorstore/inmemory"
+ "trpc.group/trpc-go/trpc-agent-go/model"
+ openaimodel "trpc.group/trpc-go/trpc-agent-go/model/openai"
+ "trpc.group/trpc-go/trpc-agent-go/runner"
+ sessioninmemory "trpc.group/trpc-go/trpc-agent-go/session/inmemory"
+ "trpc.group/trpc-go/trpc-agent-go/tool"
+)
+
+const (
+ defaultModelName = "deepseek-chat"
+ defaultQuery = "What are Large Language Models and how do they work?"
+)
+
+func main() {
+ var query string
+ flag.StringVar(&query, "query", defaultQuery, "Query to ask the knowledge base")
+ flag.Parse()
+
+ ctx := context.Background()
+
+ modelName := getEnvOrDefault("MODEL_NAME", defaultModelName)
+
+ fmt.Println("🧠 Basic Knowledge Chat Demo")
+ fmt.Printf("Model: %s\n", modelName)
+ fmt.Println(strings.Repeat("=", 50))
+
+ // 1. Create file source
+ src := file.New(
+ []string{"../exampledata/file/llm.md"},
+ )
+
+ // 2. Create vector store (in-memory)
+ vs := inmemory.New()
+
+ // 3. Create embedder (OpenAI)
+ emb := openai.New()
+
+ // 4. Create knowledge base
+ kb := knowledge.New(
+ knowledge.WithVectorStore(vs),
+ knowledge.WithEmbedder(emb),
+ knowledge.WithSources([]source.Source{src}),
+ )
+
+ // 5. Load knowledge base
+ if err := kb.Load(ctx); err != nil {
+ log.Fatalf("Failed to load knowledge: %v", err)
+ }
+
+ // 6. Create knowledge search tool
+ searchTool := knowledgetool.NewKnowledgeSearchTool(kb)
+
+ // 7. Create agent with tools
+ agent := llmagent.New(
+ "basic-assistant",
+ llmagent.WithModel(openaimodel.New(modelName)),
+ llmagent.WithTools([]tool.Tool{searchTool}),
+ llmagent.WithGenerationConfig(model.GenerationConfig{
+ Temperature: floatPtr(0.7),
+ MaxTokens: intPtr(1000),
+ }),
+ )
+
+ // 8. Create runner
+ r := runner.NewRunner(
+ "basic-chat",
+ agent,
+ runner.WithSessionService(sessioninmemory.NewSessionService()),
+ )
+ defer r.Close()
+
+ // 9. Run query
+ fmt.Printf("\n💬 Query: %s\n", query)
+ fmt.Println(strings.Repeat("=", 50))
+
+ eventChan, err := r.Run(ctx, "user", "session-1", model.NewUserMessage(query))
+ if err != nil {
+ log.Fatalf("❌ Error: %v", err)
+ }
+
+ // Process events and print tool calls/responses
+ var fullResponse strings.Builder
+ for evt := range eventChan {
+ util.PrintEventWithToolCalls(evt)
+
+ if len(evt.Response.Choices) == 0 {
+ continue
+ }
+
+ choice := evt.Response.Choices[0]
+
+ // Collect streaming content
+ if choice.Delta.Content != "" {
+ fullResponse.WriteString(choice.Delta.Content)
+ }
+
+ // Print final response
+ if evt.IsFinalResponse() {
+ if fullResponse.Len() > 0 {
+ fmt.Printf("\n🤖 Final Answer:\n%s\n", fullResponse.String())
+ } else if choice.Message.Content != "" {
+ fmt.Printf("\n🤖 Final Answer:\n%s\n", choice.Message.Content)
+ }
+ }
+ }
+
+ fmt.Println("\n✅ Done!")
+}
+
+func floatPtr(f float64) *float64 { return &f }
+func intPtr(i int) *int { return &i }
+
+func getEnvOrDefault(key, defaultValue string) string {
+ if value := os.Getenv(key); value != "" {
+ return value
+ }
+ return defaultValue
+}
diff --git a/examples/knowledge/features/agentic-filter/README.md b/examples/knowledge/features/agentic-filter/README.md
new file mode 100644
index 000000000..194f29e98
--- /dev/null
+++ b/examples/knowledge/features/agentic-filter/README.md
@@ -0,0 +1,29 @@
+# Agentic Filter Example
+
+Demonstrates intelligent metadata-based filtering using LLM.
+
+## Features
+
+- **Automatic filter selection**: LLM chooses appropriate metadata filters based on query
+- **Metadata-aware search**: Filter results by category, topic, content type, etc.
+- **Improved accuracy**: More relevant results by filtering before semantic search
+
+## How it works
+
+1. Sources are tagged with metadata (category, topic, etc.)
+2. Agent analyzes user query
+3. LLM selects appropriate metadata filters
+4. Search is performed only on filtered documents
+
+## Run
+
+```bash
+export OPENAI_API_KEY=your-api-key
+go run main.go
+```
+
+## Example queries
+
+- "Find programming-related content" → filters by `topic=programming`
+- "Show me machine learning docs" → filters by `topic=machine_learning`
+- "What's in golang content?" → filters by `content_type=golang`
diff --git a/examples/knowledge/features/agentic-filter/main.go b/examples/knowledge/features/agentic-filter/main.go
new file mode 100644
index 000000000..25da5fae0
--- /dev/null
+++ b/examples/knowledge/features/agentic-filter/main.go
@@ -0,0 +1,128 @@
+//
+// Tencent is pleased to support the open source community by making trpc-agent-go available.
+//
+// Copyright (C) 2025 Tencent. All rights reserved.
+//
+// trpc-agent-go is licensed under the Apache License Version 2.0.
+//
+//
+
+// Package main demonstrates agentic filter for intelligent metadata-based search.
+//
+// Required environment variables:
+// - OPENAI_API_KEY: Your OpenAI API key for LLM and embeddings
+// - OPENAI_BASE_URL: (Optional) Custom OpenAI API endpoint, defaults to https://api.openai.com/v1
+// - MODEL_NAME: (Optional) Model name to use, defaults to deepseek-chat
+//
+// Example usage:
+//
+// export OPENAI_API_KEY=sk-xxxx
+// export OPENAI_BASE_URL=https://api.openai.com/v1
+// export MODEL_NAME=deepseek-chat
+// go run main.go
+package main
+
+import (
+ "context"
+ "fmt"
+ "log"
+
+ "trpc.group/trpc-go/trpc-agent-go/agent/llmagent"
+ util "trpc.group/trpc-go/trpc-agent-go/examples/knowledge"
+ "trpc.group/trpc-go/trpc-agent-go/knowledge"
+ "trpc.group/trpc-go/trpc-agent-go/knowledge/embedder/openai"
+ "trpc.group/trpc-go/trpc-agent-go/knowledge/source"
+ "trpc.group/trpc-go/trpc-agent-go/knowledge/source/file"
+ knowledgetool "trpc.group/trpc-go/trpc-agent-go/knowledge/tool"
+ "trpc.group/trpc-go/trpc-agent-go/knowledge/vectorstore/inmemory"
+ "trpc.group/trpc-go/trpc-agent-go/model"
+ openaimodel "trpc.group/trpc-go/trpc-agent-go/model/openai"
+ "trpc.group/trpc-go/trpc-agent-go/runner"
+ sessioninmemory "trpc.group/trpc-go/trpc-agent-go/session/inmemory"
+ "trpc.group/trpc-go/trpc-agent-go/tool"
+)
+
+var (
+ defaultModelName = "deepseek-chat"
+)
+
+func main() {
+ ctx := context.Background()
+ modelName := util.GetEnvOrDefault("MODEL_NAME", defaultModelName)
+
+ fmt.Println("🎯 Agentic Filter Demo")
+ fmt.Println("======================")
+ fmt.Printf("Model: %s\n", modelName)
+
+ // Create sources with rich metadata
+ sources := []source.Source{
+ file.New(
+ []string{"../../exampledata/file/llm.md"},
+ file.WithName("LLM Docs"),
+ file.WithMetadataValue("category", "documentation"),
+ file.WithMetadataValue("topic", "machine_learning"),
+ file.WithMetadataValue("content_type", "llm"),
+ ),
+ file.New(
+ []string{"../../exampledata/file/golang.md"},
+ file.WithName("Golang Docs"),
+ file.WithMetadataValue("category", "documentation"),
+ file.WithMetadataValue("topic", "programming"),
+ file.WithMetadataValue("content_type", "golang"),
+ ),
+ }
+
+ // Create knowledge base
+ kb := knowledge.New(
+ knowledge.WithVectorStore(inmemory.New()),
+ knowledge.WithEmbedder(openai.New()),
+ knowledge.WithSources(sources),
+ )
+
+ if err := kb.Load(ctx); err != nil {
+ log.Fatalf("Failed to load: %v", err)
+ }
+
+ // Create agentic filter search tool and set metadata info from sources
+ knowledgeSearchTool := knowledgetool.NewAgenticFilterSearchTool(kb, source.GetAllMetadata(sources))
+
+ // Create agent with tool configured explicitly
+ agent := llmagent.New(
+ "filter-assistant",
+ llmagent.WithModel(openaimodel.New(modelName)),
+ llmagent.WithTools([]tool.Tool{knowledgeSearchTool}),
+ )
+
+ // Create runner
+ r := runner.NewRunner(
+ "filter-chat",
+ agent,
+ runner.WithSessionService(sessioninmemory.NewSessionService()),
+ )
+ defer r.Close()
+
+ // Test queries with different metadata filters
+ queries := []string{
+ "Find programming-related content",
+ "Show me machine learning documentation",
+ "What's in the golang content?",
+ }
+
+ for i, q := range queries {
+ fmt.Printf("\n%d. 🔍 Query: %s\n", i+1, q)
+ eventChan, err := r.Run(ctx, "user", fmt.Sprintf("session-%d", i),
+ model.NewUserMessage(q))
+ if err != nil {
+ log.Printf("Query failed: %v", err)
+ continue
+ }
+
+ fmt.Print(" 🤖 Response: ")
+ for evt := range eventChan {
+ util.PrintEventWithToolCalls(evt)
+ if evt.IsFinalResponse() && len(evt.Response.Choices) > 0 {
+ fmt.Println(evt.Response.Choices[0].Message.Content)
+ }
+ }
+ }
+}
diff --git a/examples/knowledge/features/management/README.md b/examples/knowledge/features/management/README.md
new file mode 100644
index 000000000..f78de50ea
--- /dev/null
+++ b/examples/knowledge/features/management/README.md
@@ -0,0 +1,87 @@
+# Knowledge Base Management Example
+
+This example demonstrates knowledge base management operations including adding, removing, reloading sources, and searching.
+
+## Features
+
+- **Source Management**: Add, remove, and reload document sources dynamically
+- **Incremental Sync**: Smart change detection with automatic orphan cleanup
+- **Metadata Support**: Attach custom metadata to sources for filtering
+- **Search**: Query the knowledge base with relevance scoring
+
+## Environment Configuration
+
+```bash
+export OPENAI_API_KEY=your_openai_api_key
+export OPENAI_BASE_URL=https://api.openai.com/v1 # Optional
+```
+
+## Usage
+
+```bash
+cd examples/knowledge/management
+go run main.go
+```
+
+## Demo Operations
+
+The demo automatically demonstrates:
+
+1. **Create Knowledge Base** - Initialize with an initial source (LLMDocs)
+2. **AddSource** - Dynamically add a new source (GolangDocs)
+3. **Search** - Query across all sources
+4. **ReloadSource** - Reload a source with updated metadata
+5. **RemoveSource** - Remove a source from the knowledge base
+6. **Search** - Verify changes after removal
+
+## Example Output
+
+```bash
+❯ go run main.go
+📚 Knowledge Management Demo
+============================
+
+1️⃣ Creating knowledge base with initial source...
+ ✅ Initial source loaded
+ Sources: 1, Total documents: 23
+ - LLMDocs: 23 docs, metadata: map[category:documentation topic:llm]
+
+2️⃣ Adding new source (GolangDocs)...
+ ✅ Source added successfully
+ Sources: 2, Total documents: 28
+ - LLMDocs: 23 docs, metadata: map[category:documentation topic:llm]
+ - GolangDocs: 5 docs, metadata: map[category:documentation topic:programming]
+
+3️⃣ Searching for 'machine learning'...
+ Found 2 results:
+ 1. [LLMDocs] score=0.449: ally available, or that the naturally occurring data is of insufficient quality....
+ 2. [LLMDocs] score=0.433: which text humans prefer. Then, the LLM can be fine-tuned through reinforcement...
+
+4️⃣ Reloading source (LLMDocs)...
+ ✅ Source reloaded with new metadata
+ Sources: 2, Total documents: 28
+ - LLMDocs: 23 docs, metadata: map[category:documentation topic:llm version:v2]
+ - GolangDocs: 5 docs, metadata: map[category:documentation topic:programming]
+
+5️⃣ Removing source (GolangDocs)...
+ ✅ Source removed
+ Sources: 1, Total documents: 23
+ - LLMDocs: 23 docs, metadata: map[category:documentation topic:llm version:v2]
+
+6️⃣ Searching after removal...
+ Found 2 results:
+ 1. [LLMDocs] score=0.302: odel to process relationships between all elements in a sequence simultaneously,...
+ 2. [LLMDocs] score=0.292: ies inherent in human language corpora, but they also inherit inaccuracies and b...
+
+✅ Demo completed!
+```
+
+## Key APIs Demonstrated
+
+- `knowledge.New()` - Create knowledge base instance
+- `kb.Load()` - Load all configured sources
+- `kb.AddSource()` - Add a new source dynamically
+- `kb.ReloadSource()` - Reload a source (with optional metadata update)
+- `kb.RemoveSource()` - Remove a source by name
+- `kb.Search()` - Search the knowledge base
+- `kb.ShowDocumentInfo()` - Get document statistics
diff --git a/examples/knowledge/features/management/main.go b/examples/knowledge/features/management/main.go
new file mode 100644
index 000000000..362626acc
--- /dev/null
+++ b/examples/knowledge/features/management/main.go
@@ -0,0 +1,176 @@
+//
+// Tencent is pleased to support the open source community by making trpc-agent-go available.
+//
+// Copyright (C) 2025 Tencent. All rights reserved.
+//
+// trpc-agent-go is licensed under the Apache License Version 2.0.
+//
+//
+
+// Package main demonstrates knowledge management operations.
+//
+// Required environment variables:
+// - OPENAI_API_KEY: Your OpenAI API key for LLM and embeddings
+// - OPENAI_BASE_URL: (Optional) Custom OpenAI API endpoint, defaults to https://api.openai.com/v1
+// - MODEL_NAME: (Optional) Model name to use, defaults to deepseek-chat
+//
+// Example usage:
+//
+// export OPENAI_API_KEY=sk-xxxx
+// export OPENAI_BASE_URL=https://api.openai.com/v1
+// export MODEL_NAME=deepseek-chat
+// go run main.go
+package main
+
+import (
+ "context"
+ "fmt"
+ "log"
+ "strings"
+
+ "trpc.group/trpc-go/trpc-agent-go/knowledge"
+ openaiembedder "trpc.group/trpc-go/trpc-agent-go/knowledge/embedder/openai"
+ "trpc.group/trpc-go/trpc-agent-go/knowledge/source"
+ "trpc.group/trpc-go/trpc-agent-go/knowledge/source/file"
+ vectorinmemory "trpc.group/trpc-go/trpc-agent-go/knowledge/vectorstore/inmemory"
+)
+
+func main() {
+ ctx := context.Background()
+
+ fmt.Println("📚 Knowledge Management Demo")
+ fmt.Println("============================")
+
+ // Step 1: Create initial knowledge base with one source
+ fmt.Println("\n1️⃣ Creating knowledge base with initial source...")
+ llmSource := file.New(
+ []string{"../exampledata/file/llm.md"},
+ file.WithName("LLMDocs"),
+ file.WithMetadata(map[string]any{"topic": "llm", "category": "documentation"}),
+ )
+
+ kb := knowledge.New(
+ knowledge.WithEmbedder(openaiembedder.New()),
+ knowledge.WithVectorStore(vectorinmemory.New()),
+ knowledge.WithSources([]source.Source{llmSource}),
+ knowledge.WithEnableSourceSync(true),
+ )
+
+ if err := kb.Load(ctx); err != nil {
+ log.Fatalf("Failed to load: %v", err)
+ }
+ fmt.Println(" ✅ Initial source loaded")
+ showSources(ctx, kb)
+
+ // Step 2: Add a new source dynamically
+ fmt.Println("\n2️⃣ Adding new source (GolangDocs)...")
+ golangSource := file.New(
+ []string{"../exampledata/file/golang.md"},
+ file.WithName("GolangDocs"),
+ file.WithMetadata(map[string]any{"topic": "programming", "category": "documentation"}),
+ )
+
+ if err := kb.AddSource(ctx, golangSource); err != nil {
+ log.Printf(" ❌ Failed to add source: %v", err)
+ } else {
+ fmt.Println(" ✅ Source added successfully")
+ }
+ showSources(ctx, kb)
+
+ // Step 3: Search across all sources
+ fmt.Println("\n3️⃣ Searching for 'machine learning'...")
+ result, err := kb.Search(ctx, &knowledge.SearchRequest{
+ Query: "machine learning",
+ MaxResults: 2,
+ })
+ if err != nil {
+ log.Printf(" ❌ Search failed: %v", err)
+ } else {
+ printSearchResults(result)
+ }
+
+ // Step 4: Reload a source (simulate content update)
+ fmt.Println("\n4️⃣ Reloading source (LLMDocs)...")
+ reloadSource := file.New(
+ []string{"../exampledata/file/llm.md"},
+ file.WithName("LLMDocs"),
+ file.WithMetadata(map[string]any{"topic": "llm", "category": "documentation", "version": "v2"}),
+ )
+ if err := kb.ReloadSource(ctx, reloadSource); err != nil {
+ log.Printf(" ❌ Failed to reload: %v", err)
+ } else {
+ fmt.Println(" ✅ Source reloaded with new metadata")
+ }
+ showSources(ctx, kb)
+
+ // Step 5: Remove a source
+ fmt.Println("\n5️⃣ Removing source (GolangDocs)...")
+ if err := kb.RemoveSource(ctx, "GolangDocs"); err != nil {
+ log.Printf(" ❌ Failed to remove: %v", err)
+ } else {
+ fmt.Println(" ✅ Source removed")
+ }
+ showSources(ctx, kb)
+
+ // Step 6: Search again to verify
+ fmt.Println("\n6️⃣ Searching after removal...")
+ result, err = kb.Search(ctx, &knowledge.SearchRequest{
+ Query: "programming concepts",
+ MaxResults: 2,
+ })
+ if err != nil {
+ log.Printf(" ❌ Search failed: %v", err)
+ } else {
+ printSearchResults(result)
+ }
+
+ fmt.Println("\n✅ Demo completed!")
+}
+
+func showSources(ctx context.Context, kb *knowledge.BuiltinKnowledge) {
+ docInfos, err := kb.ShowDocumentInfo(ctx)
+ if err != nil {
+ fmt.Printf(" Error: %v\n", err)
+ return
+ }
+
+ // Count documents per source
+ sourceCounts := make(map[string]int)
+ sourceMetadata := make(map[string]map[string]any)
+ for _, info := range docInfos {
+ sourceCounts[info.SourceName]++
+ if sourceMetadata[info.SourceName] == nil {
+ sourceMetadata[info.SourceName] = filterInternalMetadata(info.AllMeta)
+ }
+ }
+
+ fmt.Printf(" Sources: %d, Total documents: %d\n", len(sourceCounts), len(docInfos))
+ for name, count := range sourceCounts {
+ fmt.Printf(" - %s: %d docs, metadata: %v\n", name, count, sourceMetadata[name])
+ }
+}
+
+func printSearchResults(result *knowledge.SearchResult) {
+ fmt.Printf(" Found %d results:\n", len(result.Documents))
+ for i, doc := range result.Documents {
+ sourceName := ""
+ if name, ok := doc.Document.Metadata[source.MetaSourceName].(string); ok {
+ sourceName = name
+ }
+ content := doc.Document.Content
+ if len(content) > 80 {
+ content = content[:80] + "..."
+ }
+ fmt.Printf(" %d. [%s] score=%.3f: %s\n", i+1, sourceName, doc.Score, content)
+ }
+}
+
+func filterInternalMetadata(metadata map[string]any) map[string]any {
+ filtered := make(map[string]any)
+ for k, v := range metadata {
+ if !strings.HasPrefix(k, source.MetaPrefix) {
+ filtered[k] = v
+ }
+ }
+ return filtered
+}
diff --git a/examples/knowledge/features/metadata-filter/README.md b/examples/knowledge/features/metadata-filter/README.md
new file mode 100644
index 000000000..9c2473a98
--- /dev/null
+++ b/examples/knowledge/features/metadata-filter/README.md
@@ -0,0 +1,47 @@
+# Metadata Filter Example
+
+Demonstrates programmatic metadata filtering for precise search control.
+
+## Features
+
+- **Equal filter**: Exact metadata matching
+- **OR filter**: Match any condition
+- **AND filter**: Match all conditions
+- **NOT filter**: Exclude matches
+- **Complex filters**: Nested combinations
+
+## Run
+
+```bash
+export OPENAI_API_KEY=your-api-key
+go run main.go
+```
+
+## Filter types
+
+```go
+// Equal
+searchfilter.Equal("topic", "programming")
+
+// OR
+searchfilter.Or(
+ searchfilter.Equal("topic", "programming"),
+ searchfilter.Equal("topic", "machine_learning"),
+)
+
+// AND
+searchfilter.And(
+ searchfilter.Equal("topic", "programming"),
+ searchfilter.Equal("difficulty", "beginner"),
+)
+
+// NOT
+searchfilter.Not(searchfilter.Equal("topic", "advanced"))
+```
+
+## Use cases
+
+- Filter by document type, category, date range
+- Combine multiple filter conditions
+- Exclude irrelevant content
+- Fine-grained search control
diff --git a/examples/knowledge/features/metadata-filter/main.go b/examples/knowledge/features/metadata-filter/main.go
new file mode 100644
index 000000000..dadb7e883
--- /dev/null
+++ b/examples/knowledge/features/metadata-filter/main.go
@@ -0,0 +1,141 @@
+//
+// Tencent is pleased to support the open source community by making trpc-agent-go available.
+//
+// Copyright (C) 2025 Tencent. All rights reserved.
+//
+// trpc-agent-go is licensed under the Apache License Version 2.0.
+//
+//
+
+// Package main demonstrates programmatic metadata filtering.
+//
+// Required environment variables:
+// - OPENAI_API_KEY: Your OpenAI API key for LLM and embeddings
+// - OPENAI_BASE_URL: (Optional) Custom OpenAI API endpoint, defaults to https://api.openai.com/v1
+// - MODEL_NAME: (Optional) Model name to use, defaults to deepseek-chat
+//
+// Example usage:
+//
+// export OPENAI_API_KEY=sk-xxxx
+// export OPENAI_BASE_URL=https://api.openai.com/v1
+// export MODEL_NAME=deepseek-chat
+// go run main.go
+package main
+
+import (
+ "context"
+ "fmt"
+ "log"
+ "strings"
+
+ "trpc.group/trpc-go/trpc-agent-go/knowledge"
+ "trpc.group/trpc-go/trpc-agent-go/knowledge/embedder/openai"
+ "trpc.group/trpc-go/trpc-agent-go/knowledge/searchfilter"
+ "trpc.group/trpc-go/trpc-agent-go/knowledge/source"
+ "trpc.group/trpc-go/trpc-agent-go/knowledge/source/file"
+ "trpc.group/trpc-go/trpc-agent-go/knowledge/vectorstore/inmemory"
+)
+
+func main() {
+ ctx := context.Background()
+
+ fmt.Println("🔎 Metadata Filter Demo")
+ fmt.Println("=======================")
+
+ // Create sources with metadata
+ sources := []source.Source{
+ file.New(
+ []string{"../../exampledata/file/llm.md"},
+ file.WithName("LLM Docs"),
+ file.WithMetadataValue("topic", "machine_learning"),
+ file.WithMetadataValue("difficulty", "advanced"),
+ ),
+ file.New(
+ []string{"../../exampledata/file/golang.md"},
+ file.WithName("Golang Docs"),
+ file.WithMetadataValue("topic", "programming"),
+ file.WithMetadataValue("difficulty", "beginner"),
+ ),
+ }
+
+ // Create knowledge base
+ kb := knowledge.New(
+ knowledge.WithVectorStore(inmemory.New()),
+ knowledge.WithEmbedder(openai.New()),
+ knowledge.WithSources(sources),
+ )
+
+ if err := kb.Load(ctx); err != nil {
+ log.Fatalf("Failed to load: %v", err)
+ }
+
+ // Example 1: Simple equality filter
+ // Note: Use "metadata." prefix for metadata fields in filter conditions
+ fmt.Println("\n1️⃣ Filter: metadata.topic=programming")
+ result, err := kb.Search(ctx, &knowledge.SearchRequest{
+ Query: "programming concepts",
+ MaxResults: 5,
+ SearchFilter: &knowledge.SearchFilter{
+ FilterCondition: searchfilter.Equal("metadata.topic", "programming"),
+ },
+ })
+ if err != nil {
+ log.Printf("Search failed: %v", err)
+ } else {
+ printResults(result)
+ }
+
+ // Example 2: OR filter
+ fmt.Println("\n2️⃣ Filter: metadata.topic=programming OR metadata.topic=machine_learning")
+ result, err = kb.Search(ctx, &knowledge.SearchRequest{
+ Query: "advanced topics",
+ MaxResults: 5,
+ SearchFilter: &knowledge.SearchFilter{
+ FilterCondition: searchfilter.Or(
+ searchfilter.Equal("metadata.topic", "programming"),
+ searchfilter.Equal("metadata.topic", "machine_learning"),
+ ),
+ },
+ })
+ if err != nil {
+ log.Printf("Search failed: %v", err)
+ } else {
+ printResults(result)
+ }
+
+ // Example 3: AND filter
+ fmt.Println("\n3️⃣ Filter: metadata.topic=programming AND metadata.difficulty=beginner")
+ result, err = kb.Search(ctx, &knowledge.SearchRequest{
+ Query: "basics",
+ MaxResults: 5,
+ SearchFilter: &knowledge.SearchFilter{
+ FilterCondition: searchfilter.And(
+ searchfilter.Equal("metadata.topic", "programming"),
+ searchfilter.Equal("metadata.difficulty", "beginner"),
+ ),
+ },
+ })
+ if err != nil {
+ log.Printf("Search failed: %v", err)
+ } else {
+ printResults(result)
+ }
+}
+
+func printResults(result *knowledge.SearchResult) {
+ fmt.Printf(" Found %d results:\n", len(result.Documents))
+ for i, doc := range result.Documents {
+ fmt.Printf(" %d. %s (score: %.3f)\n", i+1, doc.Document.Name, doc.Score)
+ fmt.Printf(" Metadata: %v\n", filterInternalMetadata(doc.Document.Metadata))
+ }
+}
+
+func filterInternalMetadata(metadata map[string]any) map[string]any {
+ filtered := make(map[string]any)
+ for k, v := range metadata {
+ if !strings.HasPrefix(k, source.MetaPrefix) {
+ filtered[k] = v
+ }
+ }
+ return filtered
+}
diff --git a/examples/knowledge/management/README.md b/examples/knowledge/management/README.md
deleted file mode 100644
index 01612bacd..000000000
--- a/examples/knowledge/management/README.md
+++ /dev/null
@@ -1,358 +0,0 @@
-# Knowledge Base Management Example
-
-This example demonstrates a comprehensive knowledge base management system that supports multiple document sources and vector stores.
-
-## Features
-
-- **Multiple Vector Store Support**: Elasticsearch, TCVector, PGVector, InMemory
-- **Multiple Embedder Support**: OpenAI, Gemini embedding models
-- **File Source Management**: Automatic document parsing, chunking, and metadata extraction
-- **Real-time Synchronization**: Detects file changes and updates knowledge base
-- **Metadata Filtering**: Advanced filtering by document metadata and tags
-- **Smart Cleanup**: Automatic removal of orphaned documents
-- **Interactive CLI**: User-friendly console interface
-- **Batch Processing**: High-performance document ingestion with concurrency control
-
-## Data Directory Structure
-
-```
-examples/knowledge/management/
-├── data/
-│ ├── golang.md # Default Golang documentation
-│ ├── llm.md # Default LLM documentation
-│ └── other.md # Additional documentation
-├── main.go # Main application
-└── README.md # This file
-```
-
-## Environment Configuration
-
-### Required Environment Variables
-
-```bash
-# OpenAI Embedding
-export OPENAI_BASE_URL="your-openai-base-url" # Required for OpenAI model and embedder
-export OPENAI_API_KEY=your_openai_api_key
-export OPENAI_EMBEDDING_MODEL=text-embedding-3-small # optional
-
-# Elasticsearch (if using)
-export ELASTICSEARCH_HOSTS=http://localhost:9200
-export ELASTICSEARCH_USERNAME=elastic
-export ELASTICSEARCH_PASSWORD=your_password
-export ELASTICSEARCH_INDEX_NAME=trpc_agent_documents
-
-# PGVectors (if using)
-export PGVECTOR_HOST=localhost
-export PGVECTOR_PORT=5432
-export PGVECTOR_USER=root
-export PGVECTOR_PASSWORD=your_password
-export PGVECTOR_DATABASE=vectordb
-
-# TCVector (if using)
-export TCVECTOR_URL=your_tcvector_url
-export TCVECTOR_USERNAME=your_username
-export TCVECTOR_PASSWORD=your_password
-```
-
-## Usage
-
-### Installation & Setup
-
-```bash
-cd examples/knowledgemanage
-go run main.go [options]
-```
-
-### Command Line Options
-
-```bash
--embedder string
- Embedding provider (default "openai")
--vectorstore string
- Vector store backend: elasticsearch, tcvector, pgvector, inmemory (default "inmemory")
--source_sync
- Enable source sync for incremental sync (default true)
-```
-
-### Load Options
-
-The system supports several load options that serve different purposes and users:
-
-#### 1. **`WithRecreate(recreate bool)`** - **For System Administrators/Developers**
- - **Target Users**: Scenarios requiring complete knowledge base rebuild
- - **Function**: Forces complete clearance of vector store and re-imports all documents
- - **Use Cases**:
- - Initial knowledge base setup
- - Major structural changes to source files
- - Complete knowledge base reset required
-
- **Example Code**:
- ```go
- // Completely rebuild knowledge base (clear all content and re-import)
- err := knowledge.Load(ctx,
- knowledge.WithRecreate(true), // Clear and rebuild
- knowledge.WithShowProgress(true),
- )
- ```
-
-#### 2. **`WithEnableSourceSync(enable bool)`** - **For End Users/Operations Staff**
- - **Target Users**: Scenarios requiring continuous synchronization and maintenance
- - **Function**: Incremental synchronization with smart change detection and updates
- - **Use Cases**:
- - Daily document updates and maintenance
- - Avoid redundant processing of unchanged content
- - Automatic orphan document cleanup
-
- **Example Code**:
- ```go
- // Enable incremental synchronization (enabled by default)
- kb := knowledge.New(
- knowledge.WithEmbedder(embedder),
- knowledge.WithVectorStore(vectorStore),
- knowledge.WithEnableSourceSync(true), // Incremental sync
- )
-
- // New sources are automatically synchronized when added
- err := kb.AddSource(ctx, newSource)
- ```
-
-**Key Differences**:
-- `WithRecreate`: Destructive operation for initialization/reset scenarios
-- `WithEnableSourceSync`: Non-destructive operation for daily maintenance scenarios
-- Can be combined: Rebuild first then enable sync for optimal performance
-
-### Example Output
-
-```bash
-❯ go run main.go -vectorstore=tcvector
-2025/09/09 19:32:18 [Warning] Jieba will use default file for stopwords, which is /tmp/tencent/vectordatabase/data/default_stopwords.txt
-2025/09/09 19:32:18 Load the stop word dictionary: "/tmp/tencent/vectordatabase/data/default_stopwords.txt"
-2025/09/09 19:32:18 Dict files path: []
-2025/09/09 19:32:18 Warning: dict files is nil.
-2025/09/09 19:32:18 Gse dictionary loaded finished.
-2025/09/09 19:32:27 Load the stop word dictionary: "/tmp/tencent/vectordatabase/data/default_stopwords.txt"
-Knowledge base initialized (Embedder: openai, Vector Store: tcvector)
-
-=== Auto-loading default sources ===
-2025-09-09T19:32:27+08:00 INFO knowledge/default.go:856 Found 47 existing documents in vector store
-2025-09-09T19:32:27+08:00 INFO knowledge/default.go:600 Loading source 2/2: GolangDocSource (type: file)
-2025-09-09T19:32:27+08:00 INFO knowledge/default.go:600 Loading source 1/2: LLMDocSource (type: file)
-2025-09-09T19:32:27+08:00 INFO knowledge/default.go:606 Fetched 5 document(s) from source GolangDocSource
-2025-09-09T19:32:27+08:00 INFO knowledge/default.go:606 Fetched 25 document(s) from source LLMDocSource
-2025-09-09T19:32:27+08:00 INFO loader/aggregator.go:101 Processed 25/25 doc(s) | source LLMDocSource
-2025-09-09T19:32:27+08:00 INFO loader/aggregator.go:101 Processed 20/25 doc(s) | source LLMDocSource
-2025-09-09T19:32:27+08:00 INFO loader/aggregator.go:101 Processed 10/25 doc(s) | source LLMDocSource
-2025-09-09T19:32:27+08:00 INFO knowledge/default.go:611 Successfully loaded source LLMDocSource
-2025-09-09T19:32:29+08:00 INFO loader/aggregator.go:101 Processed 5/5 doc(s) | source GolangDocSource
-2025-09-09T19:32:29+08:00 INFO knowledge/default.go:611 Successfully loaded source GolangDocSource
-2025-09-09T19:32:29+08:00 INFO loader/stats.go:76 Document statistics - total: 30, avg: 471.6 B, min: 154 B, max: 889 B
-2025-09-09T19:32:29+08:00 INFO loader/stats.go:87 [0, 256): 3 document(s)
-2025-09-09T19:32:29+08:00 INFO loader/stats.go:87 [256, 512): 15 document(s)
-2025-09-09T19:32:29+08:00 INFO loader/stats.go:87 [512, 1024): 12 document(s)
-2025-09-09T19:32:29+08:00 INFO knowledge/default.go:926 Starting orphan document cleanup...
-2025-09-09T19:32:29+08:00 INFO knowledge/default.go:941 Cleaning up 22 orphan/outdated documents
-2025-09-09T19:32:29+08:00 INFO knowledge/default.go:946 Successfully deleted 22 documents
-2025-09-09T19:32:29+08:00 INFO knowledge/default.go:856 Found 30 existing documents in vector store
-Default sources loaded successfully!
-
-=== Current Sources Information ===
-Total documents in vector store: 30
-
-Source Name: GolangDocSource
- Total Documents: 5
- URIs: 1 unique URI(s)
- URI: file:///data/home/xxx/workspace/trpc-agent-go/examples/knowledge/management/data/golang.md (5 documents)
- Source Metadata:
- tag: golang
-
-Source Name: LLMDocSource
- Total Documents: 25
- URIs: 1 unique URI(s)
- URI: file:///data/home/xxx/workspace/trpc-agent-go/examples/knowledge/management/data/llm.md (25 documents)
- Source Metadata:
- tag: llm
-
-Note: Use option 4 to reload all sources or option 6 to view current status
-╔════════════════════════════════════╗
-║ Knowledge Base Management System ║
-╠════════════════════════════════════╣
-║ 1. Add Source (AddSource) ║
-║ 2. Remove Source (RemoveSource) ║
-║ 3. Reload Source (ReloadSource) ║
-║ 4. Load All Sources (Load) ║
-║ 5. Search Knowledge (Search) ║
-║ 6. Show Current Sources ║
-║ 7. Exit ║
-╚════════════════════════════════════╝
-
-➤ Select operation (1-7): 1
-
-──────────────────────────────────────────────────
-Enter source name: other
-Enter file path (relative to data/ folder): ../exampledata/file/other.md
-Enter metadata (key1 value1 key2 value2 ... or press enter to skip): tag cpp
-2025-09-09T19:32:59+08:00 INFO knowledge/default.go:600 Loading source 1/1: other (type: file)
-2025-09-09T19:32:59+08:00 INFO knowledge/default.go:606 Fetched 22 document(s) from source other
-2025-09-09T19:33:00+08:00 INFO loader/aggregator.go:101 Processed 10/22 doc(s) | source other
-2025-09-09T19:33:01+08:00 INFO loader/aggregator.go:101 Processed 20/22 doc(s) | source other
-2025-09-09T19:33:01+08:00 INFO loader/aggregator.go:101 Processed 22/22 doc(s) | source other
-2025-09-09T19:33:02+08:00 INFO knowledge/default.go:611 Successfully loaded source other
-2025-09-09T19:33:02+08:00 INFO loader/stats.go:76 Document statistics - total: 22, avg: 282.8 B, min: 42 B, max: 736 B
-2025-09-09T19:33:02+08:00 INFO loader/stats.go:87 [0, 256): 10 document(s)
-2025-09-09T19:33:02+08:00 INFO loader/stats.go:87 [256, 512): 11 document(s)
-2025-09-09T19:33:02+08:00 INFO loader/stats.go:87 [512, 1024): 1 document(s)
-Successfully added source: other with metadata: map[tag:cpp]
-──────────────────────────────────────────────────
-Operation completed
-══════════════════════════════════════════════════
-
-╔════════════════════════════════════╗
-║ Knowledge Base Management System ║
-╠════════════════════════════════════╣
-║ 1. Add Source (AddSource) ║
-║ 2. Remove Source (RemoveSource) ║
-║ 3. Reload Source (ReloadSource) ║
-║ 4. Load All Sources (Load) ║
-║ 5. Search Knowledge (Search) ║
-║ 6. Show Current Sources ║
-║ 7. Exit ║
-╚════════════════════════════════════╝
-
-➤ Select operation (1-7): 6
-
-──────────────────────────────────────────────────
-
-=== Current Sources Information ===
-Total documents in vector store: 52
-
-Source Name: GolangDocSource
- Total Documents: 5
- URIs: 1 unique URI(s)
- URI: file:///data/home/xxx/workspace/trpc-agent-go/examples/knowledge/management/data/golang.md (5 documents)
- Source Metadata:
- tag: golang
-
-Source Name: other
- Total Documents: 22
- URIs: 1 unique URI(s)
- URI: file:///data/home/xxx/workspace/trpc-agent-go/examples/knowledge/management/data/other.md (22 documents)
- Source Metadata:
- tag: cpp
-
-Source Name: LLMDocSource
- Total Documents: 25
- URIs: 1 unique URI(s)
- URI: file:///data/home/xxx/workspace/trpc-agent-go/examples/knowledge/management/data/llm.md (25 documents)
- Source Metadata:
- tag: llm
-──────────────────────────────────────────────────
-Operation completed
-══════════════════════════════════════════════════
-
-╔════════════════════════════════════╗
-║ Knowledge Base Management System ║
-╠════════════════════════════════════╣
-║ 1. Add Source (AddSource) ║
-║ 2. Remove Source (RemoveSource) ║
-║ 3. Reload Source (ReloadSource) ║
-║ 4. Load All Sources (Load) ║
-║ 5. Search Knowledge (Search) ║
-║ 6. Show Current Sources ║
-║ 7. Exit ║
-╚════════════════════════════════════╝
-
-➤ Select operation (1-7): 5
-
-──────────────────────────────────────────────────
-Enter search query: golang
-Enter max results (default 1):
-2025-09-09T19:33:17+08:00 DEBUG tcvector/tcvector.go:784 tcvectordb search result: score 0.63449 id a298c2aa5abaae6e7bf7b66f6e592646 searchMode 0
-
-=== Search Results ===
-Score: 0.634
-Content: # Go Programming Language
-
-Go, also known as Golang, is an open-source programming language developed by Google in 2007 and released to the public in 2009. Created by Robert Griesemer, Rob Pike, and Ken Thompson, Go was designed to address the challenges of modern software development at scale.
-Source: GolangDocSource
-──────────────────────────────────────────────────
-Operation completed
-══════════════════════════════════════════════════
-
-╔════════════════════════════════════╗
-║ Knowledge Base Management System ║
-╠════════════════════════════════════╣
-║ 1. Add Source (AddSource) ║
-║ 2. Remove Source (RemoveSource) ║
-║ 3. Reload Source (ReloadSource) ║
-║ 4. Load All Sources (Load) ║
-║ 5. Search Knowledge (Search) ║
-║ 6. Show Current Sources ║
-║ 7. Exit ║
-╚════════════════════════════════════╝
-
-➤ Select operation (1-7):
-```
-
-### Interactive Menu
-
-The system provides an interactive CLI with the following options:
-
-1. **Add Source** - Add a new document source
-2. **Remove Source** - Remove an existing source
-3. **Reload Source** - Reload a specific source
-4. **Load All Sources** - Reload all sources (supports `WithRecreate` option for complete rebuild)
-5. **Search Knowledge** - Query the knowledge base
-6. **Show Current Sources** - Display current sources and statistics
-7. **Exit** - Exit the application
-
-### Adding a New Source
-
-When adding a source, you'll be prompted for:
-- **Source name**: Unique identifier for the source
-- **File path**: Relative path to the data file (e.g., `data/other.md`)
-- **Metadata**: Key-value pairs for filtering (e.g., `tag cpp`)
-
-**Note**: When `source_sync` is enabled (default), newly added sources are automatically synchronized with the knowledge base using incremental updates.
-
-### Search Example
-
-```bash
-➤ Select operation (1-7): 5
-Enter search query: golang
-Enter max results (default 1):
-
-=== Search Results ===
-Score: 0.634
-Content: # Go Programming Language
-
-Go, also known as Golang, is an open-source programming language...
-Source: GolangDocSource
-```
-
-### Default Sources
-
-The application automatically loads these default sources:
-
-#### File Sources (from `../exampledata/file/` directory):
-- **LLMDocSource** (`llm.md`): Large language model concepts and terminology
- - Metadata: `tag: "llm"`
-- **GolangDocSource** (`golang.md`): Go programming language documentation
- - Metadata: `tag: "golang"`
-
-#### URL Sources:
-- **Byte-pair** (urlSource1): Wikipedia article on Byte-pair encoding
- - URL: `https://en.wikipedia.org/wiki/Byte-pair_encoding`
- - Metadata: `tag: "wiki"`
- - Note: Direct URL fetching - same URL used for both content fetching and document ID generation
-
-- **trpc-go** (urlSource2): Demonstrates URL source with separate fetch and identifier URLs
- - Identifier URL: `https://trpc-go.com/Byte-pair_encoding` (used for metadata and document ID)
- - Content Fetch URL: `https://en.wikipedia.org/wiki/Byte-pair_encoding` (actual content source)
- - Metadata: `tag: "wiki"`
- - Note: This example shows how to use `WithContentFetchingURL()` to specify different URLs for content fetching vs. document identification
-
- **Notice**: When using `WithContentFetchingURL()`, the identifier URL should maintain the same path structure as the fetch URL:
- - ✅ **Correct**: If fetch URL is `http://example.com/hello.md`, identifier URL should be `http://trpc-go.com/hello.md`
- - ❌ **Incorrect**: Using `http://trpc-go.com` as identifier URL loses the document path information
- - This ensures proper document identification and metadata consistency
-
diff --git a/examples/knowledge/management/main.go b/examples/knowledge/management/main.go
deleted file mode 100644
index 8a502fecd..000000000
--- a/examples/knowledge/management/main.go
+++ /dev/null
@@ -1,552 +0,0 @@
-//
-// Tencent is pleased to support the open source community by making trpc-agent-go available.
-//
-// Copyright (C) 2025 Tencent. All rights reserved.
-//
-// trpc-agent-go is licensed under the Apache License Version 2.0.
-//
-//
-
-// Package main demonstrates knowledge management with interactive console
-package main
-
-import (
- "bufio"
- "context"
- "flag"
- "fmt"
- "os"
- "strconv"
- "strings"
-
- "trpc.group/trpc-go/trpc-agent-go/knowledge"
- "trpc.group/trpc-go/trpc-agent-go/log"
-
- // Embedder.
- "trpc.group/trpc-go/trpc-agent-go/knowledge/embedder"
- geminiembedder "trpc.group/trpc-go/trpc-agent-go/knowledge/embedder/gemini"
- openaiembedder "trpc.group/trpc-go/trpc-agent-go/knowledge/embedder/openai"
- "trpc.group/trpc-go/trpc-agent-go/knowledge/source/file"
- "trpc.group/trpc-go/trpc-agent-go/knowledge/source/url"
-
- // Source.
- "trpc.group/trpc-go/trpc-agent-go/knowledge/source"
-
- // Vector store.
- "trpc.group/trpc-go/trpc-agent-go/knowledge/vectorstore"
- vectorelasticsearch "trpc.group/trpc-go/trpc-agent-go/knowledge/vectorstore/elasticsearch"
- vectorinmemory "trpc.group/trpc-go/trpc-agent-go/knowledge/vectorstore/inmemory"
- vectorpgvector "trpc.group/trpc-go/trpc-agent-go/knowledge/vectorstore/pgvector"
- vectortcvector "trpc.group/trpc-go/trpc-agent-go/knowledge/vectorstore/tcvector"
-
- // Import PDF reader to register it (optional - comment out if PDF support is not needed).
- _ "trpc.group/trpc-go/trpc-agent-go/knowledge/document/reader/pdf"
-)
-
-// Command line flags
-var (
- embedderType = flag.String("embedder", "openai", "Embedder type: openai")
- vectorStore = flag.String("vectorstore", "inmemory", "Vector store type: inmemory/pgvector/tcvector/elasticsearch")
- sourceSync = flag.Bool("source_sync", true, "Enable source sync for incremental sync")
-)
-
-// Default values for configurations
-const (
- defaultEmbeddingModel = "text-embedding-3-small"
-)
-
-// Environment variables
-var (
- openaiEmbeddingModel = getEnvOrDefault("OPENAI_EMBEDDING_MODEL", defaultEmbeddingModel)
- // PGVector.
- pgvectorHost = getEnvOrDefault("PGVECTOR_HOST", "127.0.0.1")
- pgvectorPort = getEnvOrDefault("PGVECTOR_PORT", "5432")
- pgvectorUser = getEnvOrDefault("PGVECTOR_USER", "root")
- pgvectorPassword = getEnvOrDefault("PGVECTOR_PASSWORD", "")
- pgvectorDatabase = getEnvOrDefault("PGVECTOR_DATABASE", "vectordb")
-
- // TCVector.
- tcvectorURL = getEnvOrDefault("TCVECTOR_URL", "")
- tcvectorUsername = getEnvOrDefault("TCVECTOR_USERNAME", "")
- tcvectorPassword = getEnvOrDefault("TCVECTOR_PASSWORD", "")
-
- // Elasticsearch.
- elasticsearchHosts = getEnvOrDefault("ELASTICSEARCH_HOSTS", "http://localhost:9200")
- elasticsearchUsername = getEnvOrDefault("ELASTICSEARCH_USERNAME", "elastic")
- elasticsearchPassword = getEnvOrDefault("ELASTICSEARCH_PASSWORD", "")
- elasticsearchAPIKey = getEnvOrDefault("ELASTICSEARCH_API_KEY", "")
- elasticsearchIndexName = getEnvOrDefault("ELASTICSEARCH_INDEX_NAME", "trpc_agent_documents")
- esVersion = getEnvOrDefault("ELASTICSEARCH_VERSION", "v8")
-)
-
-// knowledgeChat manages knowledge base and chat functionality
-type knowledgeChat struct {
- source []source.Source
- knowledge *knowledge.BuiltinKnowledge
- vectorStore vectorstore.VectorStore // Store reference to vector store
- ctx context.Context
-}
-
-func main() {
- log.SetLevel(log.LevelDebug)
- flag.Parse()
-
- // Create and run the knowledge chat
- chat := &knowledgeChat{}
-
- if err := chat.run(); err != nil {
- log.Fatalf("Knowledge chat failed: %v", err)
- }
-}
-
-// run runs the console
-func (chat *knowledgeChat) run() error {
- if err := chat.setupKnowledgeBase(); err != nil {
- return fmt.Errorf("Failed to setup knowledge base: %v", err)
- }
-
- fmt.Printf("Knowledge base initialized (Embedder: %s, Vector Store: %s)\n",
- *embedderType, *vectorStore)
-
- // Auto-load and display default sources
- fmt.Println("\n=== Auto-loading default sources ===")
- if err := chat.knowledge.Load(chat.ctx,
- knowledge.WithShowProgress(true),
- knowledge.WithShowStats(true),
- knowledge.WithSourceConcurrency(2)); err != nil {
- log.Warnf("Warning: Failed to auto-load sources: %v", err)
- } else {
- fmt.Println("Default sources loaded successfully!")
-
- // Show current sources information
- chat.showCurrentSources()
-
- fmt.Println("\nNote: Use option 4 to reload all sources or option 6 to view current status")
- }
-
- return chat.runInteractiveConsole()
-}
-
-// runInteractiveConsole starts interactive console
-func (chat *knowledgeChat) runInteractiveConsole() error {
- scanner := bufio.NewScanner(os.Stdin)
-
- for {
- fmt.Println("╔════════════════════════════════════╗")
- fmt.Println("║ Knowledge Base Management System ║")
- fmt.Println("╠════════════════════════════════════╣")
- fmt.Println("║ 1. Add Source (AddSource) ║")
- fmt.Println("║ 2. Remove Source (RemoveSource) ║")
- fmt.Println("║ 3. Reload Source (ReloadSource) ║")
- fmt.Println("║ 4. Load All Sources (Load) ║")
- fmt.Println("║ 5. Search Knowledge (Search) ║")
- fmt.Println("║ 6. Show Current Sources ║")
- fmt.Println("║ 7. Exit ║")
- fmt.Println("╚════════════════════════════════════╝")
- fmt.Print("\n➤ Select operation (1-7): ")
- scanner.Scan()
- choice := strings.TrimSpace(scanner.Text())
-
- fmt.Println("\n" + strings.Repeat("─", 50))
- switch choice {
- case "1":
- chat.addSourceMenu(scanner)
- case "2":
- chat.removeSourceMenu(scanner)
- case "3":
- chat.reloadSourceMenu(scanner)
- case "4":
- chat.loadAllSources(scanner)
- case "5":
- chat.searchMenu(scanner)
- case "6":
- chat.showCurrentSources()
- case "7":
- return nil
- default:
- fmt.Println("Invalid choice, please try again")
- }
- fmt.Println(strings.Repeat("─", 50))
- fmt.Println("Operation completed")
- fmt.Println(strings.Repeat("═", 50))
- fmt.Println()
- }
-}
-
-// addSourceMenu handles adding source
-func (chat *knowledgeChat) addSourceMenu(scanner *bufio.Scanner) {
- fmt.Print("Enter source name: ")
- scanner.Scan()
- name := strings.TrimSpace(scanner.Text())
-
- fmt.Print("Enter file path (relative to data/ folder): ")
- scanner.Scan()
- filePath := strings.TrimSpace(scanner.Text())
-
- // Get metadata from user
- fmt.Print("Enter metadata (key1 value1 key2 value2 ... or press enter to skip): ")
- scanner.Scan()
- metadataInput := strings.TrimSpace(scanner.Text())
-
- // Parse metadata
- metadata := make(map[string]any)
- if metadataInput != "" {
- parts := strings.Fields(metadataInput)
- if len(parts)%2 != 0 {
- fmt.Println("Warning: Metadata should be in key-value pairs, ignoring last key")
- }
- for i := 0; i < len(parts)-1; i += 2 {
- key := parts[i]
- value := parts[i+1]
- metadata[key] = value
- }
- }
-
- // Create file source with name and metadata
- options := []file.Option{file.WithName(name)}
- if len(metadata) > 0 {
- options = append(options, file.WithMetadata(metadata))
- }
- fsSource := file.New([]string{filePath}, options...)
-
- // Add to knowledge base
- if err := chat.knowledge.AddSource(chat.ctx, fsSource,
- knowledge.WithShowProgress(true),
- knowledge.WithShowStats(true)); err != nil {
- log.Warnf("Failed to add source: %v", err)
- } else {
- fmt.Printf("Successfully added source: %s", name)
- if len(metadata) > 0 {
- fmt.Printf(" with metadata: %v", metadata)
- }
- fmt.Println()
- }
-}
-
-// removeSourceMenu handles removing source
-func (chat *knowledgeChat) removeSourceMenu(scanner *bufio.Scanner) {
- fmt.Print("Enter source name to remove: ")
- scanner.Scan()
- name := strings.TrimSpace(scanner.Text())
-
- if err := chat.knowledge.RemoveSource(chat.ctx, name); err != nil {
- log.Warnf("Failed to remove source: %v", err)
- } else {
- fmt.Printf("Successfully removed source: %s\n", name)
- }
-}
-
-// reloadSourceMenu handles reloading source
-func (chat *knowledgeChat) reloadSourceMenu(scanner *bufio.Scanner) {
- fmt.Print("Enter source name to reload: ")
- scanner.Scan()
- name := strings.TrimSpace(scanner.Text())
-
- // Find existing source - in a real implementation you would track sources
- // For demonstration, we'll create a new source with same path
-
- fmt.Print("Enter file path for reload: ")
- scanner.Scan()
- filePath := strings.TrimSpace(scanner.Text())
-
- fsSource := file.New([]string{filePath}, file.WithName(name))
-
- if err := chat.knowledge.ReloadSource(chat.ctx, fsSource,
- knowledge.WithShowProgress(true),
- knowledge.WithShowStats(true)); err != nil {
- log.Warnf("Failed to reload source: %v", err)
- } else {
- fmt.Printf("Successfully reloaded source: %s\n", name)
- }
-}
-
-// loadAllSources loads all sources
-func (chat *knowledgeChat) loadAllSources(scanner *bufio.Scanner) {
- recreate := false
- fmt.Println("Loading all sources...")
- fmt.Println("Do you want to recreate the knowledge base? (y/n)")
- scanner.Scan()
- recreateText := strings.TrimSpace(scanner.Text())
- if recreateText == "y" {
- recreate = true
- fmt.Println("Recreating knowledge base...")
- }
- if err := chat.knowledge.Load(chat.ctx,
- knowledge.WithShowProgress(true),
- knowledge.WithShowStats(true),
- knowledge.WithSourceConcurrency(2),
- knowledge.WithRecreate(recreate)); err != nil {
- log.Warnf("Failed to load sources: %v", err)
- } else {
- fmt.Println("All sources loaded successfully!")
- }
-}
-
-// searchMenu handles search
-func (chat *knowledgeChat) searchMenu(scanner *bufio.Scanner) {
- fmt.Print("Enter search query: ")
- scanner.Scan()
- query := strings.TrimSpace(scanner.Text())
-
- if query == "" {
- fmt.Println("Search query cannot be empty")
- return
- }
-
- fmt.Print("Enter max results (default 1): ")
- scanner.Scan()
- limitStr := strings.TrimSpace(scanner.Text())
- limit := 1
- if limitStr != "" {
- if l, err := strconv.Atoi(limitStr); err == nil && l > 0 {
- limit = l
- }
- }
-
- // Execute search
- result, err := chat.knowledge.Search(chat.ctx, &knowledge.SearchRequest{
- Query: query,
- MaxResults: limit,
- })
-
- if err != nil {
- log.Warnf("Search failed: %v", err)
- return
- }
-
- fmt.Printf("\n=== Search Results ===\n")
- fmt.Printf("Score: %.3f\n", result.Score)
- fmt.Printf("Content: %s\n", result.Text)
- if result.Document != nil && result.Document.Metadata != nil {
- if sourceName, ok := result.Document.Metadata[source.MetaSourceName].(string); ok {
- fmt.Printf("Source: %s\n", sourceName)
- }
- }
-}
-
-// showCurrentSources shows current sources with metadata and counts using ShowDocumentInfo
-func (chat *knowledgeChat) showCurrentSources() {
- fmt.Println("\n=== Current Sources Information ===")
-
- // Get document info using ShowDocumentInfo method
- docInfos, err := chat.knowledge.ShowDocumentInfo(chat.ctx)
- if err != nil {
- fmt.Printf("Error getting document info: %v\n", err)
- return
- }
-
- fmt.Printf("Total documents in vector store: %d\n", len(docInfos))
-
- // Organize document info by source name
- sourceStats := make(map[string]struct {
- uriCounts map[string]int // URI -> document count
- metadata map[string]any // source-level metadata
- uris []string // unique URIs for this source
- })
-
- for _, docInfo := range docInfos {
- sourceName := docInfo.SourceName
- sourceURI := docInfo.URI
-
- if sourceName != "" && sourceURI != "" {
- // Initialize source entry if not exists
- if _, exists := sourceStats[sourceName]; !exists {
- sourceStats[sourceName] = struct {
- uriCounts map[string]int
- metadata map[string]any
- uris []string
- }{
- uriCounts: make(map[string]int),
- metadata: make(map[string]any),
- uris: []string{},
- }
- }
-
- stats := sourceStats[sourceName]
-
- // Track URI counts
- stats.uriCounts[sourceURI]++
-
- // Collect unique URIs
- found := false
- for _, uri := range stats.uris {
- if uri == sourceURI {
- found = true
- break
- }
- }
- if !found {
- stats.uris = append(stats.uris, sourceURI)
- }
-
- // Collect source-level metadata (excluding system fields)
- for key, value := range docInfo.AllMeta {
- if !strings.HasPrefix(key, "trpc_agent_go_") {
- stats.metadata[key] = value
- }
- }
-
- sourceStats[sourceName] = stats
- }
- }
-
- // Display sources information
- if len(sourceStats) == 0 {
- fmt.Println("No sources found in vector store")
- return
- }
-
- for sourceName, stats := range sourceStats {
- totalDocs := 0
- for _, count := range stats.uriCounts {
- totalDocs += count
- }
-
- fmt.Printf("\nSource Name: %s\n", sourceName)
- fmt.Printf(" Total Documents: %d\n", totalDocs)
- fmt.Printf(" URIs: %d unique URI(s)\n", len(stats.uris))
-
- // Display URIs and their document counts
- for _, uri := range stats.uris {
- fmt.Printf(" URI: %s (%d documents)\n", uri, stats.uriCounts[uri])
- }
-
- if len(stats.metadata) > 0 {
- fmt.Printf(" Source Metadata:\n")
- for key, value := range stats.metadata {
- fmt.Printf(" %s: %v\n", key, value)
- }
- } else {
- fmt.Printf(" No source metadata\n")
- }
- }
-}
-
-// setupKnowledgeBase sets up knowledge base
-func (chat *knowledgeChat) setupKnowledgeBase() error {
- chat.ctx = context.Background()
-
- // Create embedder
- embedder, err := chat.setupEmbedder(chat.ctx)
- if err != nil {
- return fmt.Errorf("Failed to setup embedder: %v", err)
- }
-
- vs, err := chat.setupVectorDB()
- if err != nil {
- return fmt.Errorf("Failed to setup vector store: %v", err)
- }
-
- fileSource1 := file.New(
- []string{"../exampledata/file/llm.md"},
- file.WithName("LLMDocSource"),
- file.WithMetadata(map[string]any{"tag": "llm"}),
- )
- fileSource2 := file.New(
- []string{"../exampledata/file/golang.md"},
- file.WithName("GolangDocSource"),
- file.WithMetadata(map[string]any{"tag": "golang"}),
- )
- urlSource1 := url.New(
- []string{"https://en.wikipedia.org/wiki/Byte-pair_encoding"},
- url.WithName("Byte-pair"),
- url.WithMetadataValue("tag", "wiki"),
- )
- urlSource2 := url.New(
- []string{"https://trpc-go.com/Byte-pair_encoding"}, // contentFetchURL is configured, this url will be used to generate meta data and docID
- url.WithName("trpc-go"),
- url.WithContentFetchingURL([]string{"https://en.wikipedia.org/wiki/Byte-pair_encoding"}), // real url that fetching data
- url.WithMetadataValue("tag", "wiki"),
- )
-
- chat.source = []source.Source{fileSource1, fileSource2, urlSource1, urlSource2}
- // Create knowledge base
- chat.knowledge = knowledge.New(
- knowledge.WithEmbedder(embedder),
- knowledge.WithVectorStore(vs),
- knowledge.WithSources([]source.Source{fileSource1, fileSource2, urlSource1, urlSource2}),
- knowledge.WithEnableSourceSync(*sourceSync),
- )
-
- // Save reference to vector store
- chat.vectorStore = vs
-
- return nil
-}
-
-// setupVectorDB creates the appropriate vector store based on the selected type.
-func (chat *knowledgeChat) setupVectorDB() (vectorstore.VectorStore, error) {
- switch strings.ToLower(*vectorStore) {
- case "inmemory":
- return vectorinmemory.New(), nil
- case "pgvector":
- port, err := strconv.Atoi(pgvectorPort)
- if err != nil {
- return nil, fmt.Errorf("failed to parse pgvector port: %w", err)
- }
- vs, err := vectorpgvector.New(
- vectorpgvector.WithHost(pgvectorHost),
- vectorpgvector.WithPort(port),
- vectorpgvector.WithUser(pgvectorUser),
- vectorpgvector.WithPassword(pgvectorPassword),
- vectorpgvector.WithDatabase(pgvectorDatabase),
- )
- if err != nil {
- return nil, fmt.Errorf("failed to create pgvector store: %w", err)
- }
- return vs, nil
- case "tcvector":
- vs, err := vectortcvector.New(
- vectortcvector.WithURL(tcvectorURL),
- vectortcvector.WithUsername(tcvectorUsername),
- vectortcvector.WithPassword(tcvectorPassword),
- vectortcvector.WithCollection("tcvector-agent-go"),
- )
- if err != nil {
- return nil, fmt.Errorf("failed to create tcvector store: %w", err)
- }
- return vs, nil
- case "elasticsearch":
- // Parse hosts string into slice.
- hosts := strings.Split(elasticsearchHosts, ",")
- for i, host := range hosts {
- hosts[i] = strings.TrimSpace(host)
- }
-
- vs, err := vectorelasticsearch.New(
- vectorelasticsearch.WithAddresses(hosts),
- vectorelasticsearch.WithUsername(elasticsearchUsername),
- vectorelasticsearch.WithPassword(elasticsearchPassword),
- vectorelasticsearch.WithAPIKey(elasticsearchAPIKey),
- vectorelasticsearch.WithIndexName(elasticsearchIndexName),
- vectorelasticsearch.WithMaxRetries(3),
- vectorelasticsearch.WithVersion(esVersion),
- )
- if err != nil {
- return nil, fmt.Errorf("failed to create elasticsearch store: %w", err)
- }
- return vs, nil
- default:
- return nil, fmt.Errorf("unsupported vector store type: %s", *vectorStore)
- }
-}
-
-// setupEmbedder creates embedder based on the configured embedderType.
-func (chat *knowledgeChat) setupEmbedder(ctx context.Context) (embedder.Embedder, error) {
- switch strings.ToLower(*embedderType) {
- case "gemini":
- return geminiembedder.New(ctx)
- default: // openai
- return openaiembedder.New(
- openaiembedder.WithModel(openaiEmbeddingModel),
- ), nil
- }
-}
-
-// getEnvOrDefault returns environment variable or default value
-func getEnvOrDefault(key, defaultValue string) string {
- if value := os.Getenv(key); value != "" {
- return value
- }
- return defaultValue
-}
diff --git a/examples/knowledge/simple/README.md b/examples/knowledge/simple/README.md
deleted file mode 100644
index d62b754ac..000000000
--- a/examples/knowledge/simple/README.md
+++ /dev/null
@@ -1,615 +0,0 @@
-# Knowledge Integration Example
-
-This example demonstrates how to integrate a knowledge base with the LLM agent in `trpc-agent-go`.
-
-## Features
-
-- **Multiple Vector Store Support**: Choose between in-memory, pgvector (PostgreSQL), tcvector, or Elasticsearch storage backends
-- **Elasticsearch Version Support**: Multi-version compatibility (v7, v8, v9) with automatic version selection
-- **Multiple Embedder Support**: OpenAI and Gemini embedder options
-- **Rich Knowledge Sources**: Supports file, directory, URL, and auto-detection sources
-- **Agentic Filtered Search**: Intelligent metadata-based filtering using natural language queries
-- **Interactive Chat Interface**: Features knowledge search with multi-turn conversation support
-- **Streaming Response**: Real-time streaming of LLM responses with tool execution feedback
-- **Session Management**: Maintains conversation history and supports new session creation
-
-## Knowledge Sources Loaded
-
-The following sources are automatically loaded when you run `main.go`:
-
-| Source Type | Name / File | What It Covers | Metadata Filters |
-|-------------|-------------|----------------|------------------|
-| File | `./data/llm.md` | Large-Language-Model (LLM) basics | `category`: documentation
`topic`: machine_learning
`source_type`: local_file
`content_type`: llm |
-| File | `./data/golang.md` | Go programming language documentation | `category`: documentation
`topic`: programming
`source_type`: local_file
`content_type`: golang |
-| Directory | `./dir/` | Various documents in the directory | `category`: dataset
`topic`: machine_learning
`source_type`: local_directory
`content_type`: transformer |
-| URL | | Byte-pair encoding (BPE) algorithm | `category`: encyclopedia
`topic`: natural_language_processing
`source_type`: web_url
`content_type`: wiki |
-| Auto Source | Mixed content (Cloud computing, N-gram Wikipedia, README) | Cloud computing overview and N-gram language models | `category`: mixed
`topic`: technology
`source_type`: auto_detect
`content_type`: mixed |
-
-These documents are embedded and indexed with their metadata, enabling both `knowledge_search` and `knowledge_search_with_filter` tools to answer related questions.
-
-### Try Asking Questions Like
-
-```
-• What is a Large Language Model?
-• Tell me about Golang programming language
-• How does Byte-pair encoding work?
-• What is cloud computing?
-• Show me documentation about machine learning
-• Find content from wiki sources
-```
-
-### Agentic Filtered Search
-
-The knowledge base supports intelligent filtering through natural language queries. The LLM automatically identifies relevant metadata filters and applies them to narrow down search results.
-
-#### Natural Language Filter Examples
-
-You can use natural language to trigger intelligent filtering:
-
-```
-👤 You: Query something about programming and golang related stuff
-🤖 Assistant: I'll search for programming and golang related content in the knowledge base for you.
-🔧 Tool calls initiated:
- • knowledge_search_with_filter (ID: toolu_bdrk_01NRTD4d4o4UxJGxcYm5nJmF)
- Args: {"query": "programming golang", "filters": [{"key":"topic","value":"programming"},{"key":"content_type","value":"golang"}]}
-
-🔄 Executing tools...
-✅ Tool response (ID: toolu_bdrk_01NRTD4d4o4UxJGxcYm5nJmF): {"text":"# Go Programming Language\n\nGo, also known as Golang, is an open-source programming language developed by Google in 2007 and released to the public in 2009. Created by Robert Griesemer, Rob Pike, and Ken Thompson, Go was designed to address the challenges of modern software development at scale.","score":0.5876700282096863,"message":"Found relevant content (score: 0.59)"}
-```
-
-#### How It Works
-
-1. **Automatic Filter Detection**: The LLM analyzes your query and identifies relevant metadata filters
-2. **Intelligent Value Selection**: For each filter key, the system either:
- - Uses explicit key=value pairs if provided
- - Intelligently selects appropriate values based on context
-3. **Fallback Strategy**: If filtered search returns no results, it automatically retries without filters
-4. **Natural Language Interface**: No need to memorize filter syntax - just ask naturally
-
-## Usage
-
-### Prerequisites
-
-1. **Set OpenAI API Key** (Required for OpenAI model and embedder)
-
- ```bash
- export OPENAI_API_KEY="your-openai-api-key"
- ```
-
-2. **Configure Vector Store** (Optional - defaults to in-memory)
-
- For persistent storage, configure the appropriate environment variables for your chosen vector store.
-
-### Running the Example
-
-```bash
-cd examples/knowledge
-
-# Use in-memory vector store (default)
-go run main.go
-
-# Use PostgreSQL with pgvector
-go run main.go -vectorstore=pgvector
-
-# Use TcVector
-go run main.go -vectorstore=tcvector
-
-# Use Elasticsearch, default version v9
-go run main.go -vectorstore=elasticsearch
-
-# Use Elasticsearch with specific version (v7, v8, v9)
-go run main.go -vectorstore=elasticsearch -es-version=v8
-
-# Use Elasticsearch (or other persistent storage) and skip loading with -load=false
-go run main.go -vectorstore=elasticsearch -es-version=v7 -load=false
-
-# Specify a different model
-go run main.go -model="gpt-4o-mini" -vectorstore=pgvector
-
-# Use Gemini embedder
-go run main.go -embedder=gemini
-
-# Disable streaming mode
-go run main.go -streaming=false
-
-# Disable agentic filter (use basic knowledge search only)
-go run main.go -agentic_filter=false
-```
-
-### Interactive Commands
-
-- **Regular chat**: Type your questions naturally
-- **`/history`**: Show conversation history
-- **`/new`**: Start a new session
-- **`/exit`**: End the conversation
-
-## Available Tools
-
-| Tool | Description | Example Usage |
-| ---------------------------- | -------------------------------------------------- | ------------------------------------------------ |
-| `knowledge_search` | Search the knowledge base for relevant information | "What is a Large Language Model?" |
-| `knowledge_search_with_filter` | Intelligent filtered search with metadata-based filtering | "Query something about programming and golang related stuff" |
-
-## Vector Store Options
-
-### In-Memory (Default)
-
-- **Pros**: No external dependencies, fast for small datasets
-- **Cons**: Data doesn't persist between runs
-- **Use case**: Development, testing, small knowledge bases
-
-### PostgreSQL with pgvector
-
-- **Use case**: Production deployments, persistent storage
-- **Setup**: Requires PostgreSQL with pgvector extension
-- **Environment Variables**:
- ```bash
- export PGVECTOR_HOST="127.0.0.1"
- export PGVECTOR_PORT="5432"
- export PGVECTOR_USER="postgres"
- export PGVECTOR_PASSWORD="your_password"
- export PGVECTOR_DATABASE="vectordb"
- ```
-
-### TcVector
-
-- **Use case**: Cloud deployments, managed vector storage
-- **Setup**: Requires TcVector service credentials
-- **Environment Variables**:
- ```bash
- export TCVECTOR_URL="your_tcvector_service_url"
- export TCVECTOR_USERNAME="your_username"
- export TCVECTOR_PASSWORD="your_password"
- ```
-
-### Elasticsearch
-
-- **Use case**: Persistent search with hybrid vector + keyword retrieval, supports multiple versions (v7, v8, v9).
-- **Setup**: Requires a running Elasticsearch cluster.
-- **Version Support**:
- - **v7**: Compatible with Elasticsearch 7.x clusters
- - **v8**: Compatible with Elasticsearch 8.x clusters
- - **v9**: Compatible with Elasticsearch 9.x clusters (default)
-- **Environment Variables**:
- ```bash
- export ELASTICSEARCH_HOSTS="http://localhost:9200"
- export ELASTICSEARCH_USERNAME="" # Optional
- export ELASTICSEARCH_PASSWORD="" # Optional
- export ELASTICSEARCH_API_KEY="" # Optional
- export ELASTICSEARCH_INDEX_NAME="trpc_agent_documents"
- ```
-- **Start local Elasticsearch clusters via Docker**:
-
- ```bash
- # Elasticsearch v7
- docker run -d --name elasticsearch-v7 -p 9200:9200 \
- -e discovery.type=single-node -e xpack.security.enabled=false \
- docker.elastic.co/elasticsearch/elasticsearch:7.17.0
-
- # Elasticsearch v8
- docker run -d --name elasticsearch-v8 -p 9200:9200 \
- -e discovery.type=single-node -e xpack.security.enabled=false \
- docker.elastic.co/elasticsearch/elasticsearch:8.11.0
-
- # Elasticsearch v9 (default)
- docker run -d --name elasticsearch-v9 -p 9200:9200 \
- -e discovery.type=single-node -e xpack.security.enabled=false \
- docker.elastic.co/elasticsearch/elasticsearch:9.1.0
- ```
-
-## Embedder Options
-
-### OpenAI Embedder (Default)
-
-- **Model**: `text-embedding-3-small` (configurable)
-- **Environment Variable**: `OPENAI_EMBEDDING_MODEL`
-- **Use case**: High-quality embeddings with OpenAI's latest models
-
-### Gemini Embedder
-
-- **Model**: Uses Gemini's default embedding model
-- **Use case**: Alternative to OpenAI, good for Google ecosystem integration
-
-## Configuration
-
-### Required Environment Variables
-
-#### For OpenAI (Model + Embedder)
-
-```bash
-export OPENAI_API_KEY="your-openai-api-key" # Required for OpenAI model and embedder
-export OPENAI_BASE_URL="your-openai-base-url" # Required for OpenAI model and embedder
-export OPENAI_EMBEDDING_MODEL="text-embedding-3-small" # Required for OpenAI embedder only
-```
-
-#### For Gemini Embedder
-
-```bash
-export GOOGLE_API_KEY="your-google-api-key" # Only this is needed for Gemini embedder
-```
-
-### Optional Configuration
-
-- Vector store specific variables (see vector store documentation for details)
-- **Performance tuning**: The knowledge package provides intelligent defaults for concurrency. Adjust `WithSourceConcurrency()` and `WithDocConcurrency()` based on your specific needs:
- - **Speed up**: Increase values if processing is too slow and API limits allow
- - **Slow down**: Decrease values if hitting API rate limits or experiencing errors
- - **Default**: Use default values for balanced performance (recommended for most cases)
-- **Loading behavior**: Control progress logging, statistics display, and update frequency:
- - `WithShowProgress(false)`: Disable progress logging (default: true)
- - `WithShowStats(false)`: Disable statistics display (default: true)
- - `WithProgressStepSize(10)`: Set progress update frequency (default: 10)
-
-### Command Line Options
-
-```bash
--model string LLM model name (default: "deepseek-chat")
--streaming bool Enable streaming mode for responses (default: true)
--embedder string Embedder type: openai, gemini (default: "openai")
--vectorstore string Vector store type: inmemory, pgvector, tcvector, elasticsearch (default: "inmemory")
--es-version string Elasticsearch version: v7, v8, v9 (default: "v9", only used when vectorstore=elasticsearch)
--load bool Load data into the vector store on startup (default: true)
--agentic_filter bool Enable agentic filter for knowledge search (default: true)
-```
-
----
-
-For more details, see the code in `main.go`.
-
-## How It Works
-
-### 1. Knowledge Base Setup
-
-The example creates a knowledge base with configurable vector store and embedder:
-
-```go
-// Create knowledge base with configurable components
-vectorStore, err := c.setupVectorDB() // Supports inmemory, pgvector, tcvector
-embedder, err := c.setupEmbedder(ctx) // Supports openai, gemini
-
-kb := knowledge.New(
- knowledge.WithVectorStore(vectorStore),
- knowledge.WithEmbedder(embedder),
- knowledge.WithSources(sources),
-)
-
-// Load the knowledge base with optimized settings
-if err := kb.Load(
- ctx,
- knowledge.WithShowProgress(false), // Disable progress logging (default: true)
- knowledge.WithProgressStepSize(10), // Progress update frequency (default: 10)
- knowledge.WithShowStats(false), // Disable statistics display (default: true)
- knowledge.WithSourceConcurrency(4), // Process 4 sources concurrently
- knowledge.WithDocConcurrency(64), // Process 64 documents concurrently
- knowledge.WithRecreate(true), // **Only works with Load()** - Forces complete rebuild
-); err != nil {
- return fmt.Errorf("failed to load knowledge base: %w", err)
-}
-```
-
-### 2. Knowledge Sources
-
-The example demonstrates multiple source types:
-
-```go
-sources := []source.Source{
- // File source for local documentation
- filesource.New(
- []string{"./data/llm.md"},
- filesource.WithName("Large Language Model"),
- filesource.WithMetadataValue("category", "documentation"),
- filesource.WithMetadataValue("topic", "machine_learning"),
- filesource.WithMetadataValue("source_type", "local_file"),
- filesource.WithMetadataValue("content_type", "llm"),
- ),
-
- // Directory source for multiple files
- dirsource.New(
- []string{"./dir"},
- dirsource.WithName("Data Directory"),
- dirsource.WithMetadataValue("category", "dataset"),
- dirsource.WithMetadataValue("topic", "machine_learning"),
- dirsource.WithMetadataValue("source_type", "local_directory"),
- dirsource.WithMetadataValue("content_type", "transformer"),
- ),
-
- // URL source for web content
- urlsource.New(
- []string{"https://en.wikipedia.org/wiki/Byte-pair_encoding"},
- urlsource.WithName("Byte-pair encoding"),
- urlsource.WithMetadataValue("category", "encyclopedia"),
- urlsource.WithMetadataValue("topic", "natural_language_processing"),
- urlsource.WithMetadataValue("source_type", "web_url"),
- urlsource.WithMetadataValue("content_type", "wiki"),
- ),
-
- // Auto source handles mixed content types
- autosource.New(
- []string{
- "Cloud computing is the delivery...", // Direct text
- "https://en.wikipedia.org/wiki/N-gram", // URL
- "./README.md", // File
- },
- autosource.WithName("Mixed Content Source"),
- autosource.WithMetadataValue("category", "mixed"),
- autosource.WithMetadataValue("topic", "technology"),
- autosource.WithMetadataValue("source_type", "auto_detect"),
- autosource.WithMetadataValue("content_type", "mixed"),
- ),
-}
-```
-
-### 3. LLM Agent Configuration
-
-#### Method 1: Automatic Integration (Used in This Example)
-
-The agent is configured with the knowledge base using the `WithKnowledge()` option:
-
-```go
-// Get metadata from sources for agentic filtering
-sourcesMetadata := source.GetAllMetadata(c.sources)
-
-llmAgent := llmagent.New(
- "knowledge-assistant",
- llmagent.WithModel(modelInstance),
- llmagent.WithKnowledge(kb), // This automatically adds the knowledge_search tool
- llmagent.WithDescription("A helpful AI assistant with knowledge base access."),
- llmagent.WithInstruction("Use the knowledge_search tool to find relevant information from the knowledge base. Be helpful and conversational."),
- llmagent.WithEnableKnowledgeAgenticFilter(true), // Enable intelligent filtering
- llmagent.WithKnowledgeAgenticFilterInfo(sourcesMetadata), // Provide metadata for filtering
-)
-```
-
-#### Method 2: Manual Tool Construction (Alternative)
-
-You can also manually create knowledge search tools for more control:
-
-```go
-import (
- knowledgetool "trpc.group/trpc-go/trpc-agent-go/knowledge/tool"
-)
-
-// Option A: Basic search tool
-searchTool := knowledgetool.NewKnowledgeSearchTool(
- kb, // Knowledge instance
- knowledgetool.WithToolName("knowledge_search"),
- knowledgetool.WithToolDescription("Search for relevant information in the knowledge base."),
-)
-
-// Option B: Intelligent filter search tool
-sourcesMetadata := source.GetAllMetadata(c.sources)
-filterSearchTool := knowledgetool.NewAgenticFilterSearchTool(
- kb, // Knowledge instance
- sourcesMetadata, // Metadata information
- knowledgetool.WithToolName("knowledge_search_with_filter"),
- knowledgetool.WithToolDescription("Search the knowledge base with intelligent metadata filtering."),
-)
-
-// Create agent with manually constructed tools
-llmAgent := llmagent.New(
- "knowledge-assistant",
- llmagent.WithModel(modelInstance),
- llmagent.WithTools([]tool.Tool{searchTool}), // or filterSearchTool
- llmagent.WithDescription("A helpful AI assistant with knowledge base access."),
- llmagent.WithInstruction("Use the knowledge_search tool to find relevant information from the knowledge base."),
-)
-```
-
-**When to use manual tool construction:**
-- Multiple knowledge bases: Create separate tools for different knowledge bases
-- Custom tool names: Use descriptive names like `search_api_docs`, `search_tutorials`
-- Fine-grained control: Configure different filters for each tool
-
-### 4. Automatic Tool Registration
-
-When `WithKnowledge()` is used, the agent automatically gets access to knowledge search tools:
-
-- **`knowledge_search`**: Basic search functionality for relevant information
-- **`knowledge_search_with_filter`**: Intelligent filtered search (when agentic filter is enabled)
-- Retrieve document content based on queries with optional metadata filtering
-- Use the retrieved information to answer user questions with enhanced precision
-
-## Implementation Details
-
-### Knowledge Interface
-
-The knowledge integration uses the `knowledge.Knowledge` interface:
-
-```go
-type Knowledge interface {
- Search(ctx context.Context, query string) (*SearchResult, error)
-}
-```
-
-### BuiltinKnowledge Implementation
-
-The example uses `BuiltinKnowledge` which provides:
-
-- **Storage**: Configurable vector store (in-memory, pgvector, or tcvector)
-- **Vector Store**: Vector similarity search with multiple backends
-- **Embedder**: OpenAI or Gemini embedder for document representation
-- **Retriever**: Complete RAG pipeline with query enhancement and reranking
-
-### Knowledge Search Tools
-
-The knowledge integration automatically provides two search tools:
-
-**`knowledge_search`** (Basic search):
-- Query validation and execution
-- Result formatting with relevance scores
-- Simple keyword-based search
-
-**`knowledge_search_with_filter`** (Agentic filtered search):
-- Intelligent metadata-based filtering
-- Natural language filter detection
-- Automatic fallback to basic search if no results
-- Enhanced precision through context-aware filtering
-
-### Knowledge Loading Configuration
-
-The example configures several loading options to optimize the user experience:
-
-```go
-// Current configuration (overrides defaults)
-if err := c.kb.Load(
- ctx,
- knowledge.WithShowProgress(false), // Disable progress logging (default: true)
- knowledge.WithProgressStepSize(10), // Progress update frequency (default: 10)
- knowledge.WithShowStats(false), // Disable statistics display (default: true)
- knowledge.WithSourceConcurrency(4), // Override default: min(4, len(sources))
- knowledge.WithDocConcurrency(64), // Override default: runtime.NumCPU()
-); err != nil {
- return fmt.Errorf("failed to load knowledge base: %w", err)
-}
-```
-
-**Available Options:**
-
-- **Progress Control**: `WithShowProgress()` - Enable/disable progress logging
-- **Update Frequency**: `WithProgressStepSize()` - Control progress update intervals
-- **Statistics Display**: `WithShowStats()` - Show/hide loading statistics
-- **Concurrency Tuning**: `WithSourceConcurrency()` and `WithDocConcurrency()` - Performance optimization
-- **Complete Rebuild**: `WithRecreate()` - **Only works with Load() function** - Forces complete knowledge base rebuild by clearing vector store and re-importing all documents
-- **Incremental Sync**: `WithEnableSourceSync()` - Enables smart change detection and incremental updates (enabled by default)
-
-### Components Used
-
-- **Vector Stores**:
- - `vectorstore/inmemory`: In-memory vector store with cosine similarity
- - `vectorstore/pgvector`: PostgreSQL-based persistent vector storage
- - `vectorstore/tcvector`: TcVector cloud-native vector storage
-- **Embedders**:
- - `embedder/openai`: OpenAI embeddings API integration
- - `embedder/gemini`: Gemini embeddings API integration
-- **Sources**: `source/{file,dir,url,auto}`: Multiple content source types
-- **Session**: `session/inmemory`: In-memory conversation state management
-- **Runner**: Multi-turn conversation management with streaming support
-
-## Extending the Example
-
-### Adding Custom Sources
-
-```go
-// Add your own content sources
-customSources := []source.Source{
- filesource.New(
- []string{"/path/to/your/docs/*.md"},
- filesource.WithName("Custom Documentation"),
- ),
- urlsource.New(
- []string{"https://your-company.com/api-docs"},
- urlsource.WithMetadataValue("category", "api"),
- ),
-}
-
-// Append to existing sources
-allSources := append(sources, customSources...)
-```
-
-### Production Considerations
-
-- Use persistent vector store (`pgvector` or `tcvector`) for production
-- Secure API key management
-- Monitor vector store performance
-- Implement proper error handling and logging
-- Consider using environment-specific configuration files
-
-### Performance Optimization & API Rate Limits
-
-The example uses parallel processing to optimize knowledge base loading. The knowledge package provides intelligent defaults:
-
-```go
-// Current configuration (overrides defaults)
-knowledge.WithShowProgress(false), // Disable progress logging
-knowledge.WithSourceConcurrency(4), // Override default: min(4, len(sources))
-knowledge.WithDocConcurrency(64), // Override default: runtime.NumCPU()
-```
-
-**⚠️ Important Notes:**
-
-- **Increased API Calls**: Parallel processing will significantly increase the frequency of API requests to your embedder service (OpenAI/Gemini)
-- **Rate Limit Considerations**: Be aware of your API provider's rate limits and quotas
-- **Cost Impact**: More concurrent requests may increase costs for paid API services
-- **Adjust as Needed**: Balance between processing speed and API rate limits
-
-**Performance vs. Rate Limits - Finding the Sweet Spot:**
-
-The concurrency settings affect both processing speed and API request frequency. You need to find the right balance:
-
-- **Too Slow?** Increase concurrency for faster processing
-- **Too Fast?** Reduce concurrency to avoid hitting API rate limits
-- **Just Right?** Default values are optimized for most scenarios
-
-**When to Adjust Concurrency:**
-
-```go
-// If processing is too slow (acceptable API usage)
-knowledge.WithSourceConcurrency(8), // Increase from default 4
-knowledge.WithDocConcurrency(128), // Increase from default runtime.NumCPU()
-
-// If hitting API rate limits (slower but stable)
-knowledge.WithSourceConcurrency(2), // Reduce from default 4
-knowledge.WithDocConcurrency(16), // Reduce from default runtime.NumCPU()
-
-// Default values (balanced approach)
-// knowledge.WithSourceConcurrency() // Default: min(4, len(sources))
-// knowledge.WithDocConcurrency() // Default: runtime.NumCPU()
-```
-
-## Key Dependencies
-
-- `agent/llmagent`: LLM agent with streaming and tool support
-- `knowledge/*`: Complete RAG pipeline with multiple source types
-- `knowledge/vectorstore/*`: Multiple vector storage backends
-- `knowledge/embedder/*`: Multiple embedder implementations
-- `runner`: Multi-turn conversation management with session state
-- `session/inmemory`: In-memory session state management
-
-## Troubleshooting
-
-### Common Issues
-
-1. **OpenAI API Key Error**
-
- - Ensure `OPENAI_API_KEY` is set correctly
- - Verify your OpenAI account has embedding API access
- - **Important**: For OpenAI model and embedder, you must also set `OPENAI_BASE_URL` and `OPENAI_EMBEDDING_MODEL`
-
-2. **Gemini API Key Error**
-
- - Ensure `GOOGLE_API_KEY` is set correctly
- - Verify your Google Cloud project has Gemini API access enabled
- - **Note**: Gemini embedder only requires `GOOGLE_API_KEY`, no additional model configuration needed
-
-3. **Vector Store Connection Issues**
-
- - For pgvector: Ensure PostgreSQL is running and pgvector extension is installed
- - For tcvector: Verify service credentials and network connectivity
- - Check environment variables are set correctly
-
-4. **Knowledge Loading Errors**
-
- - Verify source files/URLs are accessible
- - Check file permissions for local sources
- - Ensure stable internet connection for URL sources
-
-5. **Embedder Configuration Issues**
-
- - **OpenAI**: Verify `OPENAI_API_KEY`, `OPENAI_BASE_URL`, and `OPENAI_EMBEDDING_MODEL` are all set
- - **Gemini**: Verify `GOOGLE_API_KEY` is set and API is enabled
- - Check API quotas and rate limits for both services
-
-6. **API Rate Limiting & Parallel Processing Issues**
-
- - **Rate Limit Errors**: If you see "rate limit exceeded" errors, reduce concurrency settings to slow down API requests
- - **High API Costs**: Parallel processing increases API call frequency - the default values are optimized for most use cases
- - **Slow Loading**: If knowledge base loading is too slow, increase concurrency (if API limits allow) for faster processing
- - **Memory Usage**: High concurrency may increase memory usage during loading - defaults are balanced for performance and resource usage
- - **Performance Tuning**: Monitor loading time vs. API errors to find optimal concurrency settings for your environment
-
-7. **Knowledge Loading Behavior Issues**
-
- - **Need More Visibility**: If you want to see loading progress, enable `WithShowProgress(true)` and `WithShowStats(true)`
- - **Too Many Logs**: If progress logs are too verbose, increase `WithProgressStepSize()` value or disable with `WithShowProgress(false)`
- - **Missing Statistics**: If you need document size and count information, enable `WithShowStats(true)`
- - **Custom Progress Frequency**: Adjust `WithProgressStepSize()` to control how often progress updates are logged (default: 10)
diff --git a/examples/knowledge/simple/main.go b/examples/knowledge/simple/main.go
deleted file mode 100644
index ccf32af8d..000000000
--- a/examples/knowledge/simple/main.go
+++ /dev/null
@@ -1,734 +0,0 @@
-//
-// Tencent is pleased to support the open source community by making trpc-agent-go available.
-//
-// Copyright (C) 2025 Tencent. All rights reserved.
-//
-// trpc-agent-go is licensed under the Apache License Version 2.0.
-//
-//
-
-// Package main demonstrates knowledge integration with the LLM agent.
-package main
-
-import (
- "bufio"
- "context"
- "encoding/json"
- "flag"
- "fmt"
- "log"
- "math"
- "os"
- "strconv"
- "strings"
- "time"
-
- "github.com/tencent/vectordatabase-sdk-go/tcvectordb"
- "trpc.group/trpc-go/trpc-agent-go/agent/llmagent"
- "trpc.group/trpc-go/trpc-agent-go/event"
- "trpc.group/trpc-go/trpc-agent-go/knowledge"
- "trpc.group/trpc-go/trpc-agent-go/knowledge/searchfilter"
- knowledgetool "trpc.group/trpc-go/trpc-agent-go/knowledge/tool"
- "trpc.group/trpc-go/trpc-agent-go/model"
- openaimodel "trpc.group/trpc-go/trpc-agent-go/model/openai"
- "trpc.group/trpc-go/trpc-agent-go/runner"
- sessioninmemory "trpc.group/trpc-go/trpc-agent-go/session/inmemory"
- "trpc.group/trpc-go/trpc-agent-go/tool"
-
- // Embedder.
- "trpc.group/trpc-go/trpc-agent-go/knowledge/document"
- "trpc.group/trpc-go/trpc-agent-go/knowledge/embedder"
- geminiembedder "trpc.group/trpc-go/trpc-agent-go/knowledge/embedder/gemini"
- huggingfaceembedder "trpc.group/trpc-go/trpc-agent-go/knowledge/embedder/huggingface"
- ollamaembedder "trpc.group/trpc-go/trpc-agent-go/knowledge/embedder/ollama"
- openaiembedder "trpc.group/trpc-go/trpc-agent-go/knowledge/embedder/openai"
-
- // Source.
- "trpc.group/trpc-go/trpc-agent-go/knowledge/source"
- autosource "trpc.group/trpc-go/trpc-agent-go/knowledge/source/auto"
- dirsource "trpc.group/trpc-go/trpc-agent-go/knowledge/source/dir"
- filesource "trpc.group/trpc-go/trpc-agent-go/knowledge/source/file"
- urlsource "trpc.group/trpc-go/trpc-agent-go/knowledge/source/url"
-
- // Vector store.
- "trpc.group/trpc-go/trpc-agent-go/knowledge/vectorstore"
- vectorelasticsearch "trpc.group/trpc-go/trpc-agent-go/knowledge/vectorstore/elasticsearch"
- vectorinmemory "trpc.group/trpc-go/trpc-agent-go/knowledge/vectorstore/inmemory"
- vectorpgvector "trpc.group/trpc-go/trpc-agent-go/knowledge/vectorstore/pgvector"
- vectortcvector "trpc.group/trpc-go/trpc-agent-go/knowledge/vectorstore/tcvector"
-
- // Import PDF reader to register it (optional - comment out if PDF support is not needed).
- _ "trpc.group/trpc-go/trpc-agent-go/knowledge/document/reader/pdf"
-)
-
-// command line flags.
-var (
- modelName = flag.String("model", "deepseek-chat", "Name of the model to use")
- streaming = flag.Bool("streaming", true, "Enable streaming mode for responses")
- embedderType = flag.String("embedder", "openai", "Embedder type: openai, gemini, ollama, huggingface")
- vectorStore = flag.String("vectorstore", "inmemory", "Vector store type: inmemory, pgvector, tcvector, elasticsearch")
- esVersion = flag.String("es-version", "v9", "Elasticsearch version: v7, v8, v9 (only used when vectorstore=elasticsearch)")
- agenticFilter = flag.Bool("agentic_filter", true, "Enable agentic filter for knowledge search")
- recreate = flag.Bool("recreate", false, "Recreate the vector store on startup, all data in vector store will be deleted.")
- sourceSync = flag.Bool("source_sync", false, "Enable source sync for incremental sync, all data in vector store will be sync with source. And orphan documents will be deleted.")
-)
-
-// Default values for optional configurations.
-const (
- defaultEmbeddingModel = "text-embedding-3-small"
-)
-
-// Environment variables for vector stores and embedder.
-var (
- // OpenAI embedding model.
- openaiEmbeddingModel = getEnvOrDefault("OPENAI_EMBEDDING_MODEL", defaultEmbeddingModel)
-
- // PGVector.
- pgvectorHost = getEnvOrDefault("PGVECTOR_HOST", "127.0.0.1")
- pgvectorPort = getEnvOrDefault("PGVECTOR_PORT", "5432")
- pgvectorUser = getEnvOrDefault("PGVECTOR_USER", "root")
- pgvectorPassword = getEnvOrDefault("PGVECTOR_PASSWORD", "")
- pgvectorDatabase = getEnvOrDefault("PGVECTOR_DATABASE", "vectordb")
-
- // TCVector.
- tcvectorURL = getEnvOrDefault("TCVECTOR_URL", "")
- tcvectorUsername = getEnvOrDefault("TCVECTOR_USERNAME", "")
- tcvectorPassword = getEnvOrDefault("TCVECTOR_PASSWORD", "")
-
- // Elasticsearch.
- elasticsearchHosts = getEnvOrDefault("ELASTICSEARCH_HOSTS", "http://localhost:9200")
- elasticsearchUsername = getEnvOrDefault("ELASTICSEARCH_USERNAME", "")
- elasticsearchPassword = getEnvOrDefault("ELASTICSEARCH_PASSWORD", "")
- elasticsearchAPIKey = getEnvOrDefault("ELASTICSEARCH_API_KEY", "")
- elasticsearchIndexName = getEnvOrDefault("ELASTICSEARCH_INDEX_NAME", "trpc_agent_go")
-)
-
-func main() {
- // Parse command line flags.
- flag.Parse()
-
- fmt.Printf("🧠 Knowledge-Enhanced Chat Demo\n")
- fmt.Printf("Model: %s\n", *modelName)
- fmt.Printf("Vector Store: %s\n", *vectorStore)
- if *vectorStore == "elasticsearch" {
- fmt.Printf(" (Version: %s)", *esVersion)
- }
- if *agenticFilter {
- fmt.Printf("Available tools: knowledge_search, knowledge_search_with_agentic_filter\n")
- } else {
- fmt.Printf("Available tools: knowledge_search\n")
- }
- fmt.Println(strings.Repeat("=", 50))
-
- // Create and run the chat.
- chat := &knowledgeChat{
- modelName: *modelName,
- embedderType: *embedderType,
- vectorStore: *vectorStore,
- }
-
- if err := chat.run(); err != nil {
- log.Fatalf("Chat failed: %v", err)
- }
-}
-
-// knowledgeChat manages the conversation with knowledge integration.
-type knowledgeChat struct {
- sources []source.Source
- modelName string
- embedderType string
- vectorStore string
- runner runner.Runner
- userID string
- sessionID string
- kb *knowledge.BuiltinKnowledge
-}
-
-// run starts the interactive chat session.
-func (c *knowledgeChat) run() error {
- ctx := context.Background()
-
- // Setup the runner with knowledge base.
- if err := c.setup(ctx); err != nil {
- return fmt.Errorf("setup failed: %w", err)
- }
-
- // Ensure runner resources are cleaned up (trpc-agent-go >= v0.5.0)
- defer c.runner.Close()
-
- // Start interactive chat.
- return c.startChat(ctx)
-}
-
-// setup creates the runner with LLM agent, knowledge base, and tools.
-func (c *knowledgeChat) setup(ctx context.Context) error {
- // Create OpenAI model.
- modelInstance := openaimodel.New(c.modelName)
-
- // Create knowledge base with sample documents.
- if err := c.setupKnowledgeBase(ctx); err != nil {
- return fmt.Errorf("failed to setup knowledge base: %w", err)
- }
-
- // Create LLM agent with knowledge.
- genConfig := model.GenerationConfig{
- MaxTokens: intPtr(2000),
- Temperature: floatPtr(0.7),
- Stream: *streaming,
- }
-
- // get all metadata from sources, it contains keys and values, llm will choose keys values for the filter
- sourcesMetadata := source.GetAllMetadata(c.sources)
-
- // You can also bind knowledge base on agent directly.
- // It's equal with set knowledge tool in agent.
- // But you can only set one knowledgebase for one agent.
-
- // llmAgent := llmagent.New(
- // agentName,
- // llmagent.WithModel(modelInstance),
- // llmagent.WithDescription("A helpful AI assistant with knowledge base access."),
- // llmagent.WithInstruction("Use the knowledge_search tool to find relevant information from the knowledge base. Be helpful and conversational."),
- // llmagent.WithGenerationConfig(genConfig),
- // llmagent.WithKnowledge(c.kb),
- // // This will automatically add the knowledge_search tool.
- // llmagent.WithEnableKnowledgeAgenticFilter(*agenticFilter),
- // llmagent.WithKnowledgeAgenticFilterInfo(sourcesMetadata),
- // )
-
- agentName := "knowledge-assistant"
- knowledgeTool := knowledgetool.NewAgenticFilterSearchTool(
- c.kb,
- sourcesMetadata,
- knowledgetool.WithToolName("knowledge_search"),
- // filter by topic = programming and content_type = llm
- knowledgetool.WithConditionedFilter(
- searchfilter.Or(
- searchfilter.Equal("topic", "programming"),
- searchfilter.Equal("content_type", "llm"),
- ),
- ),
- knowledgetool.WithToolDescription(
- "Use the knowledge_search tool to find relevant information from the knowledge base. Be helpful and conversational."),
- )
-
- llmAgent := llmagent.New(
- agentName,
- llmagent.WithModel(modelInstance),
- llmagent.WithDescription("A helpful AI assistant."),
- llmagent.WithInstruction("Use the knowledge_search tool to find relevant information from the knowledge base. Be helpful and conversational."),
- llmagent.WithGenerationConfig(genConfig),
- llmagent.WithTools([]tool.Tool{knowledgeTool}),
- )
-
- // Create session service.
- sessionService := sessioninmemory.NewSessionService()
-
- // Create runner.
- appName := "knowledge-chat"
- c.runner = runner.NewRunner(
- appName,
- llmAgent,
- runner.WithSessionService(sessionService),
- )
-
- // Setup identifiers.
- c.userID = "user"
- c.sessionID = fmt.Sprintf("knowledge-session-%d", time.Now().Unix())
-
- fmt.Printf("✅ Knowledge chat ready! Session: %s\n", c.sessionID)
- fmt.Printf("📚 Knowledge base loaded with sample documents\n\n")
-
- return nil
-}
-
-// setupVectorDB creates the appropriate vector store based on the selected type.
-func (c *knowledgeChat) setupVectorDB() (vectorstore.VectorStore, error) {
- switch strings.ToLower(*vectorStore) {
- case "inmemory":
- return vectorinmemory.New(), nil
- case "pgvector":
- port, err := strconv.Atoi(pgvectorPort)
- if err != nil {
- return nil, fmt.Errorf("failed to parse pgvector port: %w", err)
- }
- vs, err := vectorpgvector.New(
- vectorpgvector.WithHost(pgvectorHost),
- vectorpgvector.WithPort(port),
- vectorpgvector.WithUser(pgvectorUser),
- vectorpgvector.WithPassword(pgvectorPassword),
- vectorpgvector.WithDatabase(pgvectorDatabase),
- )
- if err != nil {
- return nil, fmt.Errorf("failed to create pgvector store: %w", err)
- }
- return vs, nil
- case "tcvector":
- docBuilder := func(tcDoc tcvectordb.Document) (*document.Document, []float64, error) {
- doc := &document.Document{
- ID: tcDoc.Id,
- }
- if field, ok := tcDoc.Fields["name"]; ok {
- doc.Name = field.String()
- }
- if field, ok := tcDoc.Fields["content"]; ok {
- doc.Content = field.String()
- }
- if field, ok := tcDoc.Fields["created_at"]; ok {
- u := min(field.Uint64(), uint64(math.MaxInt64))
- //nolint:gosec // u is not overflowed and the conversion is safe.
- doc.CreatedAt = time.Unix(int64(u), 0)
- }
- if field, ok := tcDoc.Fields["updated_at"]; ok {
- u := min(field.Uint64(), uint64(math.MaxInt64))
- //nolint:gosec // u is not overflowed and the conversion is safe.
- doc.UpdatedAt = time.Unix(int64(u), 0)
- }
- if field, ok := tcDoc.Fields["metadata"]; ok {
- if metadata, ok := field.Val.(map[string]any); ok {
- doc.Metadata = metadata
- }
- }
-
- embedding := make([]float64, len(tcDoc.Vector))
- for i, v := range tcDoc.Vector {
- embedding[i] = float64(v)
- }
- return doc, embedding, nil
- }
- vs, err := vectortcvector.New(
- vectortcvector.WithURL(tcvectorURL),
- vectortcvector.WithUsername(tcvectorUsername),
- vectortcvector.WithPassword(tcvectorPassword),
- vectortcvector.WithCollection("tcvector-agent-go"),
- // used for build document from tcvector query result
- vectortcvector.WithDocBuilder(docBuilder),
- )
- if err != nil {
- return nil, fmt.Errorf("failed to create tcvector store: %w", err)
- }
- return vs, nil
- case "elasticsearch":
- // Parse hosts string into slice.
- hosts := strings.Split(elasticsearchHosts, ",")
- for i, host := range hosts {
- hosts[i] = strings.TrimSpace(host)
- }
-
- docBuilder := func(hitSource json.RawMessage) (*document.Document, []float64, error) {
- var source struct {
- ID string `json:"id"`
- Name string `json:"name"`
- Content string `json:"content"`
- CreatedAt time.Time `json:"created_at"`
- UpdatedAt time.Time `json:"updated_at"`
- Embedding []float64 `json:"embedding"`
- Metadata map[string]any `json:"metadata"`
- }
- if err := json.Unmarshal(hitSource, &source); err != nil {
- return nil, nil, err
- }
- // Create document.
- doc := &document.Document{
- ID: source.ID,
- Name: source.Name,
- Content: source.Content,
- CreatedAt: source.CreatedAt,
- UpdatedAt: source.UpdatedAt,
- Metadata: source.Metadata,
- }
- return doc, source.Embedding, nil
- }
-
- vs, err := vectorelasticsearch.New(
- vectorelasticsearch.WithAddresses(hosts),
- vectorelasticsearch.WithUsername(elasticsearchUsername),
- vectorelasticsearch.WithPassword(elasticsearchPassword),
- vectorelasticsearch.WithAPIKey(elasticsearchAPIKey),
- vectorelasticsearch.WithIndexName(elasticsearchIndexName),
- vectorelasticsearch.WithMaxRetries(3),
- vectorelasticsearch.WithVersion(*esVersion),
- // Custom document builder for document retrieval. If not provided, uses default builder.
- vectorelasticsearch.WithDocBuilder(docBuilder),
- )
- if err != nil {
- return nil, fmt.Errorf("failed to create elasticsearch store: %w", err)
- }
- return vs, nil
- default:
- return nil, fmt.Errorf("unsupported vector store type: %s", *vectorStore)
- }
-}
-
-// setupEmbedder creates embedder based on the configured embedderType.
-func (c *knowledgeChat) setupEmbedder(ctx context.Context) (embedder.Embedder, error) {
- switch strings.ToLower(c.embedderType) {
- case "gemini":
- return geminiembedder.New(ctx)
- case "ollama":
- return ollamaembedder.New(), nil
- case "huggingface":
- return huggingfaceembedder.New(), nil
- default: // openai
- return openaiembedder.New(
- openaiembedder.WithModel(openaiEmbeddingModel),
- ), nil
- }
-}
-
-func (c *knowledgeChat) createSources() []source.Source {
- // Create diverse sources showcasing different types.
- sources := []source.Source{
- // File source for local documentation (if files exist).
- filesource.New(
- []string{
- "../exampledata/file/llm.md",
- },
- filesource.WithName("Large Language Model"),
- filesource.WithMetadataValue("category", "documentation"),
- filesource.WithMetadataValue("topic", "machine_learning"),
- filesource.WithMetadataValue("source_type", "local_file"),
- filesource.WithMetadataValue("content_type", "llm"),
- ),
- filesource.New(
- []string{
- "../exampledata/file/golang.md",
- },
- filesource.WithName("Golang"),
- filesource.WithMetadataValue("category", "documentation"),
- filesource.WithMetadataValue("topic", "programming"),
- filesource.WithMetadataValue("source_type", "local_file"),
- filesource.WithMetadataValue("content_type", "golang"),
- ),
- dirsource.New(
- []string{
- "../exampledata/dir",
- },
- dirsource.WithName("Data Directory"),
- dirsource.WithMetadataValue("category", "dataset"),
- dirsource.WithMetadataValue("topic", "machine_learning"),
- dirsource.WithMetadataValue("source_type", "local_directory"),
- dirsource.WithMetadataValue("content_type", "transformer"),
- ),
- // URL source for web content.
- urlsource.New(
- []string{
- "https://en.wikipedia.org/wiki/Byte-pair_encoding",
- },
- urlsource.WithName("Byte-pair encoding"),
- urlsource.WithMetadataValue("category", "encyclopedia"),
- urlsource.WithMetadataValue("topic", "natural_language_processing"),
- urlsource.WithMetadataValue("source_type", "web_url"),
- urlsource.WithMetadataValue("content_type", "wiki"),
- ),
- // Auto source that can handle mixed inputs.
- autosource.New(
- []string{
- "Cloud computing is the delivery of computing services over the internet, including servers, storage, databases, networking, software, and analytics. It provides on-demand access to shared computing resources.",
- "https://en.wikipedia.org/wiki/N-gram",
- "./README.md",
- },
- autosource.WithName("Mixed Content Source"),
- autosource.WithMetadataValue("category", "mixed"),
- autosource.WithMetadataValue("topic", "technology"),
- autosource.WithMetadataValue("source_type", "auto_detect"),
- autosource.WithMetadataValue("content_type", "mixed"),
- ),
- }
- return sources
-}
-
-// setupKnowledgeBase creates a built-in knowledge base with sample documents.
-func (c *knowledgeChat) setupKnowledgeBase(ctx context.Context) error {
- // Create sources.
- c.sources = c.createSources()
-
- // Create vector store.
- vs, err := c.setupVectorDB()
- if err != nil {
- return err
- }
-
- // Create embedder.
- emb, err := c.setupEmbedder(ctx)
- if err != nil {
- return fmt.Errorf("failed to create embedder: %w", err)
- }
-
- // Create built-in knowledge base with all components.
- c.kb = knowledge.New(
- knowledge.WithVectorStore(vs),
- knowledge.WithEmbedder(emb),
- knowledge.WithSources(c.sources),
- knowledge.WithEnableSourceSync(*sourceSync),
- )
-
- // Optionally load the knowledge base.
- if err := c.kb.Load(
- ctx,
- knowledge.WithShowProgress(false), // The default is true.
- knowledge.WithProgressStepSize(10), // The default is 10.
- knowledge.WithShowStats(false), // The default is true.
- knowledge.WithSourceConcurrency(4), // The default is min(4, len(sources)).
- knowledge.WithDocConcurrency(64), // The default is runtime.NumCPU().
- knowledge.WithRecreate(*recreate),
- ); err != nil {
- return fmt.Errorf("failed to load knowledge base: %w", err)
- }
- return nil
-}
-
-// startChat runs the interactive conversation loop.
-func (c *knowledgeChat) startChat(ctx context.Context) error {
- scanner := bufio.NewScanner(os.Stdin)
-
- fmt.Println("💡 Special commands:")
- fmt.Println(" /history - Show conversation history")
- fmt.Println(" /new - Start a new session")
- fmt.Println(" /exit - End the conversation")
- fmt.Println()
- fmt.Println("🔧 Vector Store Configuration:")
- switch *vectorStore {
- case "elasticsearch":
- fmt.Printf(" Elasticsearch: %s (Index: %s)\n", elasticsearchHosts, elasticsearchIndexName)
- if elasticsearchUsername != "" {
- fmt.Printf(" Username: %s\n", elasticsearchUsername)
- }
- case "pgvector":
- fmt.Printf(" PGVector: %s:%s/%s\n", pgvectorHost, pgvectorPort, pgvectorDatabase)
- case "tcvector":
- if tcvectorURL != "" {
- fmt.Printf(" TCVector: %s\n", tcvectorURL)
- }
- }
- fmt.Println()
- fmt.Println("🔍 Try asking questions like:")
- fmt.Println(" - What is a Large Language Model?")
- fmt.Println(" - Tell me about Golang programming language")
- fmt.Println(" - How does Byte-pair encoding work?")
- fmt.Println(" - What is cloud computing?")
- fmt.Println(" - Show me documentation about machine learning")
- fmt.Println(" - Find content from wiki sources")
- fmt.Println("")
- fmt.Println("🎯 You can also try agentic filtered searches:")
- fmt.Println(" - Query something about programming and golang related stuff")
- for {
- fmt.Print("👤 You: ")
- if !scanner.Scan() {
- break
- }
-
- userInput := strings.TrimSpace(scanner.Text())
- if userInput == "" {
- continue
- }
-
- // Handle special commands.
- switch strings.ToLower(userInput) {
- case "/exit":
- fmt.Println("👋 Goodbye!")
- return nil
- case "/history":
- userInput = "show our conversation history"
- case "/new":
- c.startNewSession()
- continue
- }
-
- // Process the user message.
- if err := c.processMessage(ctx, userInput); err != nil {
- fmt.Printf("❌ Error: %v\n", err)
- }
-
- fmt.Println() // Add spacing between turns.
- }
-
- if err := scanner.Err(); err != nil {
- return fmt.Errorf("input scanner error: %w", err)
- }
-
- return nil
-}
-
-// processMessage handles a single message exchange.
-func (c *knowledgeChat) processMessage(ctx context.Context, userMessage string) error {
- message := model.NewUserMessage(userMessage)
-
- // Run the agent through the runner.
- eventChan, err := c.runner.Run(ctx, c.userID, c.sessionID, message)
- if err != nil {
- return fmt.Errorf("failed to run agent: %w", err)
- }
-
- // Process response based on streaming mode.
- if *streaming {
- return c.processStreamingResponse(eventChan)
- } else {
- return c.processNonStreamingResponse(eventChan)
- }
-}
-
-// processStreamingResponse handles the streaming response from the agent.
-func (c *knowledgeChat) processStreamingResponse(eventChan <-chan *event.Event) error {
- fmt.Print("🤖 Assistant: ")
-
- var assistantStarted bool
- var fullContent string
-
- for event := range eventChan {
- if event == nil {
- continue
- }
-
- // Handle errors.
- if event.Error != nil {
- fmt.Printf("\n❌ Error: %s\n", event.Error.Message)
- continue
- }
-
- // Detect and display tool calls.
- if len(event.Response.Choices) > 0 && len(event.Response.Choices[0].Message.ToolCalls) > 0 {
- if assistantStarted {
- fmt.Printf("\n")
- }
- fmt.Printf("🔧 Tool calls initiated:\n")
- for _, toolCall := range event.Response.Choices[0].Message.ToolCalls {
- fmt.Printf(" • %s (ID: %s)\n", toolCall.Function.Name, toolCall.ID)
- if len(toolCall.Function.Arguments) > 0 {
- fmt.Printf(" Args: %s\n", string(toolCall.Function.Arguments))
- }
- }
- fmt.Printf("\n🔄 Executing tools...\n")
- }
-
- // Detect tool responses.
- if event.Response != nil && len(event.Response.Choices) > 0 {
- hasToolResponse := false
- for _, choice := range event.Response.Choices {
- if choice.Message.Role == model.RoleTool && choice.Message.ToolID != "" {
- fmt.Printf("✅ Tool response (ID: %s): %s\n",
- choice.Message.ToolID,
- strings.TrimSpace(choice.Message.Content))
- hasToolResponse = true
- }
- }
- if hasToolResponse {
- continue
- }
- }
-
- // Process streaming content.
- if len(event.Response.Choices) > 0 {
- choice := event.Response.Choices[0]
-
- // Handle streaming delta content.
- if choice.Delta.Content != "" {
- if !assistantStarted {
- assistantStarted = true
- }
- fmt.Print(choice.Delta.Content)
- fullContent += choice.Delta.Content
- }
- }
-
- // Check if this is the final event.
- // Don't break on tool response events (Done=true but not final assistant response).
- if event.IsFinalResponse() {
- fmt.Printf("\n")
- break
- }
- }
-
- return nil
-}
-
-// processNonStreamingResponse handles the non-streaming response from the agent.
-func (c *knowledgeChat) processNonStreamingResponse(eventChan <-chan *event.Event) error {
- fmt.Print("🤖 Assistant: ")
-
- var fullContent string
- var hasToolCalls bool
-
- for event := range eventChan {
- if event == nil {
- continue
- }
-
- // Handle errors.
- if event.Error != nil {
- fmt.Printf("\n❌ Error: %s\n", event.Error.Message)
- continue
- }
-
- // Detect and display tool calls.
- if len(event.Response.Choices) > 0 && len(event.Response.Choices[0].Message.ToolCalls) > 0 {
- if !hasToolCalls {
- fmt.Printf("\n🔧 Tool calls initiated:\n")
- hasToolCalls = true
- }
- for _, toolCall := range event.Response.Choices[0].Message.ToolCalls {
- fmt.Printf(" • %s (ID: %s)\n", toolCall.Function.Name, toolCall.ID)
- if len(toolCall.Function.Arguments) > 0 {
- fmt.Printf(" Args: %s\n", string(toolCall.Function.Arguments))
- }
- }
- fmt.Printf("\n🔄 Executing tools...\n")
- }
-
- // Detect tool responses.
- if event.Response != nil && len(event.Response.Choices) > 0 {
- hasToolResponse := false
- for _, choice := range event.Response.Choices {
- if choice.Message.Role == model.RoleTool && choice.Message.ToolID != "" {
- fmt.Printf("✅ Tool response (ID: %s): %s\n",
- choice.Message.ToolID,
- strings.TrimSpace(choice.Message.Content))
- hasToolResponse = true
- }
- }
- if hasToolResponse {
- continue
- }
- }
-
- // Process final content from non-streaming response.
- if event.IsFinalResponse() {
- // In non-streaming mode, the final content should be in the Message.Content
- if len(event.Response.Choices) > 0 {
- choice := event.Response.Choices[0]
- if choice.Message.Content != "" {
- fullContent = choice.Message.Content
- fmt.Print(fullContent)
- }
- }
- fmt.Printf("\n")
- break
- }
- }
-
- return nil
-}
-
-// startNewSession creates a new chat session.
-func (c *knowledgeChat) startNewSession() {
- c.sessionID = fmt.Sprintf("knowledge-session-%d", time.Now().Unix())
- fmt.Printf("🔄 New session started: %s\n\n", c.sessionID)
-}
-
-// Helper functions.
-
-func intPtr(i int) *int {
- return &i
-}
-
-func floatPtr(f float64) *float64 {
- return &f
-}
-
-// getEnvOrDefault returns the environment variable value or a default value if not set.
-func getEnvOrDefault(key, defaultValue string) string {
- if value := os.Getenv(key); value != "" {
- return value
- }
- return defaultValue
-}
diff --git a/examples/knowledge/sources/auto-source/README.md b/examples/knowledge/sources/auto-source/README.md
new file mode 100644
index 000000000..b12031314
--- /dev/null
+++ b/examples/knowledge/sources/auto-source/README.md
@@ -0,0 +1,27 @@
+# Auto Source Example
+
+Demonstrates automatic content type detection for mixed inputs.
+
+## Features
+
+- Automatic type detection (text/file/URL)
+- Mixed content handling
+- Simplified API for diverse sources
+
+## Run
+
+```bash
+export OPENAI_BASE_URL=xxx
+export OPENAI_API_KEY=xxx
+export MODEL_NAME=xxx
+go run main.go
+```
+
+## How it works
+
+Auto source automatically determines if the input is:
+- Plain text content
+- File path (local file)
+- URL (web page)
+
+This simplifies handling multiple content types without manual classification.
diff --git a/examples/knowledge/sources/auto-source/main.go b/examples/knowledge/sources/auto-source/main.go
new file mode 100644
index 000000000..ec1b5d7aa
--- /dev/null
+++ b/examples/knowledge/sources/auto-source/main.go
@@ -0,0 +1,123 @@
+//
+// Tencent is pleased to support the open source community by making trpc-agent-go available.
+//
+// Copyright (C) 2025 Tencent. All rights reserved.
+//
+// trpc-agent-go is licensed under the Apache License Version 2.0.
+//
+//
+
+// Package main demonstrates using auto source for mixed content types.
+//
+// Required environment variables:
+// - OPENAI_API_KEY: Your OpenAI API key for LLM and embeddings
+// - OPENAI_BASE_URL: (Optional) Custom OpenAI API endpoint, defaults to https://api.openai.com/v1
+// - MODEL_NAME: (Optional) Model name to use, defaults to deepseek-chat
+//
+// Example usage:
+//
+// export OPENAI_API_KEY=sk-xxxx
+// export OPENAI_BASE_URL=https://api.openai.com/v1
+// export MODEL_NAME=deepseek-chat
+// go run main.go
+package main
+
+import (
+ "context"
+ "fmt"
+ "log"
+
+ "trpc.group/trpc-go/trpc-agent-go/agent/llmagent"
+ util "trpc.group/trpc-go/trpc-agent-go/examples/knowledge"
+ "trpc.group/trpc-go/trpc-agent-go/knowledge"
+ "trpc.group/trpc-go/trpc-agent-go/knowledge/embedder/openai"
+ "trpc.group/trpc-go/trpc-agent-go/knowledge/source"
+ "trpc.group/trpc-go/trpc-agent-go/knowledge/source/auto"
+ knowledgetool "trpc.group/trpc-go/trpc-agent-go/knowledge/tool"
+ "trpc.group/trpc-go/trpc-agent-go/knowledge/vectorstore/inmemory"
+ "trpc.group/trpc-go/trpc-agent-go/model"
+ openaimodel "trpc.group/trpc-go/trpc-agent-go/model/openai"
+ "trpc.group/trpc-go/trpc-agent-go/runner"
+ sessioninmemory "trpc.group/trpc-go/trpc-agent-go/session/inmemory"
+ "trpc.group/trpc-go/trpc-agent-go/tool"
+)
+
+var (
+ defaultModelName = "deepseek-chat"
+)
+
+func main() {
+ ctx := context.Background()
+ modelName := util.GetEnvOrDefault("MODEL_NAME", defaultModelName)
+ fmt.Println("🔮 Auto Source Demo")
+ fmt.Println("===================")
+
+ // Auto source automatically detects content type: text, file, or URL
+ src := auto.New(
+ []string{
+ // Plain text
+ "Quantum computing uses quantum bits (qubits) to perform computations exponentially faster than classical computers for certain problems.",
+ // File path
+ "../../exampledata/file/llm.md",
+ // URL
+ "https://en.wikipedia.org/wiki/N-gram",
+ },
+ auto.WithName("Mixed Content"),
+ auto.WithMetadataValue("source_type", "auto"),
+ )
+
+ // Create knowledge base
+ kb := knowledge.New(
+ knowledge.WithVectorStore(inmemory.New()),
+ knowledge.WithEmbedder(openai.New()),
+ knowledge.WithSources([]source.Source{src}),
+ )
+
+ fmt.Println("\n📥 Loading mixed content (text, file, URL)...")
+ if err := kb.Load(ctx, knowledge.WithShowProgress(true)); err != nil {
+ log.Fatalf("Failed to load: %v", err)
+ }
+
+ // Create knowledge search tool
+ searchTool := knowledgetool.NewKnowledgeSearchTool(kb)
+
+ // Create agent
+ agent := llmagent.New(
+ "auto-assistant",
+ llmagent.WithModel(openaimodel.New(modelName)),
+ llmagent.WithTools([]tool.Tool{searchTool}),
+ )
+
+ // Create runner
+ r := runner.NewRunner(
+ "auto-chat",
+ agent,
+ runner.WithSessionService(sessioninmemory.NewSessionService()),
+ )
+ defer r.Close()
+
+ // Test queries
+ queries := []string{
+ "What are qubits?",
+ "Tell me about n-grams",
+ "What is a Large Language Model?",
+ }
+
+ for i, q := range queries {
+ fmt.Printf("\n%d. 🔍 Query: %s\n", i+1, q)
+ eventChan, err := r.Run(ctx, "user", fmt.Sprintf("session-%d", i),
+ model.NewUserMessage(q))
+ if err != nil {
+ log.Printf("Query failed: %v", err)
+ continue
+ }
+
+ fmt.Print(" 🤖 Response: ")
+ for evt := range eventChan {
+ util.PrintEventWithToolCalls(evt)
+ if evt.IsFinalResponse() && len(evt.Response.Choices) > 0 {
+ fmt.Println(evt.Response.Choices[0].Message.Content)
+ }
+ }
+ }
+}
diff --git a/examples/knowledge/sources/directory-source/README.md b/examples/knowledge/sources/directory-source/README.md
new file mode 100644
index 000000000..9a54d3a89
--- /dev/null
+++ b/examples/knowledge/sources/directory-source/README.md
@@ -0,0 +1,23 @@
+# Directory Source Example
+
+Demonstrates how to load entire directories of documents.
+
+## Features
+
+- Recursive directory loading
+- Automatic file type detection
+- Batch processing of multiple files
+- Progress tracking
+
+## Run
+
+```bash
+export OPENAI_BASE_URL=xxx
+export OPENAI_API_KEY=xxx
+export MODEL_NAME=xxx
+go run main.go
+```
+
+## How it works
+
+The directory source recursively scans the specified directory and processes all supported file formats automatically.
diff --git a/examples/knowledge/sources/directory-source/main.go b/examples/knowledge/sources/directory-source/main.go
new file mode 100644
index 000000000..5a37506d3
--- /dev/null
+++ b/examples/knowledge/sources/directory-source/main.go
@@ -0,0 +1,108 @@
+//
+// Tencent is pleased to support the open source community by making trpc-agent-go available.
+//
+// Copyright (C) 2025 Tencent. All rights reserved.
+//
+// trpc-agent-go is licensed under the Apache License Version 2.0.
+//
+//
+
+// Package main demonstrates using directory sources.
+//
+// Required environment variables:
+// - OPENAI_API_KEY: Your OpenAI API key for LLM and embeddings
+// - OPENAI_BASE_URL: (Optional) Custom OpenAI API endpoint, defaults to https://api.openai.com/v1
+// - MODEL_NAME: (Optional) Model name to use, defaults to deepseek-chat
+//
+// Example usage:
+//
+// export OPENAI_API_KEY=sk-xxxx
+// export OPENAI_BASE_URL=https://api.openai.com/v1
+// export MODEL_NAME=deepseek-chat
+// go run main.go
+package main
+
+import (
+ "context"
+ "fmt"
+ "log"
+
+ "trpc.group/trpc-go/trpc-agent-go/agent/llmagent"
+ util "trpc.group/trpc-go/trpc-agent-go/examples/knowledge"
+ "trpc.group/trpc-go/trpc-agent-go/knowledge"
+ "trpc.group/trpc-go/trpc-agent-go/knowledge/embedder/openai"
+ "trpc.group/trpc-go/trpc-agent-go/knowledge/source"
+ "trpc.group/trpc-go/trpc-agent-go/knowledge/source/dir"
+ knowledgetool "trpc.group/trpc-go/trpc-agent-go/knowledge/tool"
+ "trpc.group/trpc-go/trpc-agent-go/knowledge/vectorstore/inmemory"
+ "trpc.group/trpc-go/trpc-agent-go/model"
+ openaimodel "trpc.group/trpc-go/trpc-agent-go/model/openai"
+ "trpc.group/trpc-go/trpc-agent-go/runner"
+ sessioninmemory "trpc.group/trpc-go/trpc-agent-go/session/inmemory"
+ "trpc.group/trpc-go/trpc-agent-go/tool"
+)
+
+var (
+ defaultModelName = "deepseek-chat"
+)
+
+func main() {
+ ctx := context.Background()
+ modelName := util.GetEnvOrDefault("MODEL_NAME", defaultModelName)
+ fmt.Println("📁 Directory Source Demo")
+ fmt.Println("========================")
+
+ // Create directory source - recursively loads all supported files
+ src := dir.New(
+ []string{"../../exampledata/dir"},
+ dir.WithName("Documentation Directory"),
+ dir.WithMetadataValue("source_type", "directory"),
+ dir.WithMetadataValue("category", "docs"),
+ )
+
+ // Create knowledge base
+ kb := knowledge.New(
+ knowledge.WithVectorStore(inmemory.New()),
+ knowledge.WithEmbedder(openai.New()),
+ knowledge.WithSources([]source.Source{src}),
+ )
+
+ fmt.Println("\n📥 Loading directory contents...")
+ if err := kb.Load(ctx, knowledge.WithShowProgress(true)); err != nil {
+ log.Fatalf("Failed to load: %v", err)
+ }
+
+ // Create knowledge search tool
+ searchTool := knowledgetool.NewKnowledgeSearchTool(kb)
+
+ // Create agent
+ agent := llmagent.New(
+ "dir-assistant",
+ llmagent.WithModel(openaimodel.New(modelName)),
+ llmagent.WithTools([]tool.Tool{searchTool}),
+ )
+
+ // Create runner
+ r := runner.NewRunner(
+ "dir-chat",
+ agent,
+ runner.WithSessionService(sessioninmemory.NewSessionService()),
+ )
+ defer r.Close()
+
+ // Test query
+ fmt.Println("\n🔍 Searching knowledge base...")
+ eventChan, err := r.Run(ctx, "user", "session-1",
+ model.NewUserMessage("What topics are covered in the documentation?"))
+ if err != nil {
+ log.Fatalf("Run failed: %v", err)
+ }
+
+ fmt.Print("🤖 Response: ")
+ for evt := range eventChan {
+ util.PrintEventWithToolCalls(evt)
+ if evt.IsFinalResponse() && len(evt.Response.Choices) > 0 {
+ fmt.Println(evt.Response.Choices[0].Message.Content)
+ }
+ }
+}
diff --git a/examples/knowledge/sources/file-source/README.md b/examples/knowledge/sources/file-source/README.md
new file mode 100644
index 000000000..26dba4fcc
--- /dev/null
+++ b/examples/knowledge/sources/file-source/README.md
@@ -0,0 +1,27 @@
+# File Source Example
+
+Demonstrates how to use file sources with metadata.
+
+## Features
+
+- Multiple file sources
+- Custom metadata (category, format)
+- Support for different file formats (Markdown, PDF, etc.)
+
+## Run
+
+```bash
+export OPENAI_BASE_URL=xxx
+export OPENAI_API_KEY=xxx
+export MODEL_NAME=xxx
+go run main.go
+```
+
+## Supported formats
+
+- Markdown (`.md`)
+- PDF (`.pdf`) - requires PDF reader import
+- Text (`.txt`)
+- CSV (`.csv`)
+- JSON (`.json`)
+- DOCX (`.docx`)
diff --git a/examples/knowledge/sources/file-source/main.go b/examples/knowledge/sources/file-source/main.go
new file mode 100644
index 000000000..8fc189cfe
--- /dev/null
+++ b/examples/knowledge/sources/file-source/main.go
@@ -0,0 +1,117 @@
+//
+// Tencent is pleased to support the open source community by making trpc-agent-go available.
+//
+// Copyright (C) 2025 Tencent. All rights reserved.
+//
+// trpc-agent-go is licensed under the Apache License Version 2.0.
+//
+//
+
+// Package main demonstrates using file sources with different formats.
+//
+// Required environment variables:
+// - OPENAI_API_KEY: Your OpenAI API key for LLM and embeddings
+// - OPENAI_BASE_URL: (Optional) Custom OpenAI API endpoint, defaults to https://api.openai.com/v1
+// - MODEL_NAME: (Optional) Model name to use, defaults to deepseek-chat
+//
+// Example usage:
+//
+// export OPENAI_API_KEY=sk-xxxx
+// export OPENAI_BASE_URL=https://api.openai.com/v1
+// export MODEL_NAME=deepseek-chat
+// go run main.go
+package main
+
+import (
+ "context"
+ "fmt"
+ "log"
+
+ "trpc.group/trpc-go/trpc-agent-go/agent/llmagent"
+ util "trpc.group/trpc-go/trpc-agent-go/examples/knowledge"
+ "trpc.group/trpc-go/trpc-agent-go/knowledge"
+ "trpc.group/trpc-go/trpc-agent-go/knowledge/embedder/openai"
+ "trpc.group/trpc-go/trpc-agent-go/knowledge/source"
+ "trpc.group/trpc-go/trpc-agent-go/knowledge/source/file"
+ knowledgetool "trpc.group/trpc-go/trpc-agent-go/knowledge/tool"
+ "trpc.group/trpc-go/trpc-agent-go/knowledge/vectorstore/inmemory"
+ "trpc.group/trpc-go/trpc-agent-go/model"
+ openaimodel "trpc.group/trpc-go/trpc-agent-go/model/openai"
+ "trpc.group/trpc-go/trpc-agent-go/runner"
+ sessioninmemory "trpc.group/trpc-go/trpc-agent-go/session/inmemory"
+ "trpc.group/trpc-go/trpc-agent-go/tool"
+
+ _ "trpc.group/trpc-go/trpc-agent-go/knowledge/document/reader/pdf"
+)
+
+var (
+ defaultModelName = "deepseek-chat"
+)
+
+func main() {
+ ctx := context.Background()
+ modelName := util.GetEnvOrDefault("MODEL_NAME", defaultModelName)
+ fmt.Println("📄 File Source Demo")
+ fmt.Println("===================")
+
+ // Create multiple file sources with metadata
+ sources := []source.Source{
+ file.New(
+ []string{"../../exampledata/file/llm.md"},
+ file.WithName("LLM Docs"),
+ file.WithMetadataValue("category", "machine_learning"),
+ file.WithMetadataValue("format", "markdown"),
+ ),
+ file.New(
+ []string{"../../exampledata/file/golang.md"},
+ file.WithName("Golang Docs"),
+ file.WithMetadataValue("category", "programming"),
+ file.WithMetadataValue("format", "markdown"),
+ ),
+ }
+
+ // Create knowledge base
+ kb := knowledge.New(
+ knowledge.WithVectorStore(inmemory.New()),
+ knowledge.WithEmbedder(openai.New()),
+ knowledge.WithSources(sources),
+ )
+
+ if err := kb.Load(ctx); err != nil {
+ log.Fatalf("Failed to load: %v", err)
+ }
+
+ // Create knowledge search tool
+ searchTool := knowledgetool.NewKnowledgeSearchTool(kb)
+
+ // Create agent
+ agent := llmagent.New(
+ "file-assistant",
+ llmagent.WithModel(openaimodel.New(modelName)),
+ llmagent.WithTools([]tool.Tool{searchTool}),
+ )
+
+ // Create runner
+ r := runner.NewRunner(
+ "file-chat",
+ agent,
+ runner.WithSessionService(sessioninmemory.NewSessionService()),
+ )
+ defer r.Close()
+
+ // Test query
+ fmt.Println("\n🔍 Testing knowledge search...")
+ eventChan, err := r.Run(ctx, "user", "session-1",
+ model.NewUserMessage("What is the difference between LLMs and traditional programming?"))
+ if err != nil {
+ log.Fatalf("Run failed: %v", err)
+ }
+
+ fmt.Print("🤖 Response: ")
+ for evt := range eventChan {
+ util.PrintEventWithToolCalls(evt)
+ if evt.IsFinalResponse() && len(evt.Response.Choices) > 0 {
+ fmt.Println(evt.Response.Choices[0].Message.Content)
+ }
+ }
+}
diff --git a/examples/knowledge/sources/url-source/README.md b/examples/knowledge/sources/url-source/README.md
new file mode 100644
index 000000000..92e9efc84
--- /dev/null
+++ b/examples/knowledge/sources/url-source/README.md
@@ -0,0 +1,23 @@
+# URL Source Example
+
+Demonstrates how to fetch and index web content.
+
+## Features
+
+- Web page fetching
+- HTML to text conversion
+- Multiple URLs support
+- Custom metadata
+
+## Run
+
+```bash
+export OPENAI_BASE_URL=xxx
+export OPENAI_API_KEY=xxx
+export MODEL_NAME=xxx
+go run main.go
+```
+
+## How it works
+
+The URL source fetches web pages, extracts text content, and indexes it into the knowledge base.
diff --git a/examples/knowledge/sources/url-source/main.go b/examples/knowledge/sources/url-source/main.go
new file mode 100644
index 000000000..b4463e9d7
--- /dev/null
+++ b/examples/knowledge/sources/url-source/main.go
@@ -0,0 +1,111 @@
+//
+// Tencent is pleased to support the open source community by making trpc-agent-go available.
+//
+// Copyright (C) 2025 Tencent. All rights reserved.
+//
+// trpc-agent-go is licensed under the Apache License Version 2.0.
+//
+//
+
+// Package main demonstrates using URL sources to fetch web content.
+//
+// Required environment variables:
+// - OPENAI_API_KEY: Your OpenAI API key for LLM and embeddings
+// - OPENAI_BASE_URL: (Optional) Custom OpenAI API endpoint, defaults to https://api.openai.com/v1
+// - MODEL_NAME: (Optional) Model name to use, defaults to deepseek-chat
+//
+// Example usage:
+//
+// export OPENAI_API_KEY=sk-xxxx
+// export OPENAI_BASE_URL=https://api.openai.com/v1
+// export MODEL_NAME=deepseek-chat
+// go run main.go
+package main
+
+import (
+ "context"
+ "fmt"
+ "log"
+
+ "trpc.group/trpc-go/trpc-agent-go/agent/llmagent"
+ util "trpc.group/trpc-go/trpc-agent-go/examples/knowledge"
+ "trpc.group/trpc-go/trpc-agent-go/knowledge"
+ "trpc.group/trpc-go/trpc-agent-go/knowledge/embedder/openai"
+ "trpc.group/trpc-go/trpc-agent-go/knowledge/source"
+ "trpc.group/trpc-go/trpc-agent-go/knowledge/source/url"
+ knowledgetool "trpc.group/trpc-go/trpc-agent-go/knowledge/tool"
+ "trpc.group/trpc-go/trpc-agent-go/knowledge/vectorstore/inmemory"
+ "trpc.group/trpc-go/trpc-agent-go/model"
+ openaimodel "trpc.group/trpc-go/trpc-agent-go/model/openai"
+ "trpc.group/trpc-go/trpc-agent-go/runner"
+ sessioninmemory "trpc.group/trpc-go/trpc-agent-go/session/inmemory"
+ "trpc.group/trpc-go/trpc-agent-go/tool"
+)
+
+var (
+ defaultModelName = "deepseek-chat"
+)
+
+func main() {
+ ctx := context.Background()
+ modelName := util.GetEnvOrDefault("MODEL_NAME", defaultModelName)
+ fmt.Println("🌐 URL Source Demo")
+ fmt.Println("==================")
+
+ // Create URL source
+ src := url.New(
+ []string{
+ "https://en.wikipedia.org/wiki/Byte-pair_encoding",
+ "https://en.wikipedia.org/wiki/N-gram",
+ },
+ url.WithName("NLP Wikipedia Articles"),
+ url.WithMetadataValue("source_type", "web"),
+ url.WithMetadataValue("category", "encyclopedia"),
+ )
+
+ // Create knowledge base
+ kb := knowledge.New(
+ knowledge.WithVectorStore(inmemory.New()),
+ knowledge.WithEmbedder(openai.New()),
+ knowledge.WithSources([]source.Source{src}),
+ )
+
+ fmt.Println("\n📥 Fetching web content...")
+ if err := kb.Load(ctx, knowledge.WithShowProgress(true)); err != nil {
+ log.Fatalf("Failed to load: %v", err)
+ }
+
+ // Create knowledge search tool
+ searchTool := knowledgetool.NewKnowledgeSearchTool(kb)
+
+ // Create agent
+ agent := llmagent.New(
+ "url-assistant",
+ llmagent.WithModel(openaimodel.New(modelName)),
+ llmagent.WithTools([]tool.Tool{searchTool}),
+ )
+
+ // Create runner
+ r := runner.NewRunner(
+ "url-chat",
+ agent,
+ runner.WithSessionService(sessioninmemory.NewSessionService()),
+ )
+ defer r.Close()
+
+ // Test query
+ fmt.Println("\n🔍 Asking about fetched content...")
+ eventChan, err := r.Run(ctx, "user", "session-1",
+ model.NewUserMessage("Explain byte-pair encoding in simple terms"))
+ if err != nil {
+ log.Fatalf("Run failed: %v", err)
+ }
+
+ fmt.Print("🤖 Response: ")
+ for evt := range eventChan {
+ util.PrintEventWithToolCalls(evt)
+ if evt.IsFinalResponse() && len(evt.Response.Choices) > 0 {
+ fmt.Println(evt.Response.Choices[0].Message.Content)
+ }
+ }
+}
diff --git a/examples/knowledge/util.go b/examples/knowledge/util.go
new file mode 100644
index 000000000..79c0d0121
--- /dev/null
+++ b/examples/knowledge/util.go
@@ -0,0 +1,149 @@
+package util
+
+import (
+ "encoding/json"
+ "fmt"
+ "log"
+ "os"
+ "strconv"
+ "strings"
+
+ "trpc.group/trpc-go/trpc-agent-go/event"
+ "trpc.group/trpc-go/trpc-agent-go/knowledge/vectorstore"
+ "trpc.group/trpc-go/trpc-agent-go/knowledge/vectorstore/elasticsearch"
+ "trpc.group/trpc-go/trpc-agent-go/knowledge/vectorstore/inmemory"
+ "trpc.group/trpc-go/trpc-agent-go/knowledge/vectorstore/pgvector"
+ "trpc.group/trpc-go/trpc-agent-go/knowledge/vectorstore/tcvector"
+)
+
+// VectorStoreType defines the type of vector store.
+type VectorStoreType string
+
+const (
+ VectorStoreInMemory VectorStoreType = "inmemory"
+ VectorStorePGVector VectorStoreType = "pgvector"
+ VectorStoreTCVector VectorStoreType = "tcvector"
+ VectorStoreElasticsearch VectorStoreType = "elasticsearch"
+)
+
+// NewVectorStore creates a vector store based on the type.
+// Environment variables:
+// - VECTOR_STORE_TYPE: inmemory, pgvector, tcvector, elasticsearch (default: inmemory)
+// - PGVECTOR_HOST, PGVECTOR_PORT, PGVECTOR_USER, PGVECTOR_PASSWORD, PGVECTOR_DATABASE
+// - TCVECTOR_URL, TCVECTOR_USERNAME, TCVECTOR_PASSWORD
+// - ELASTICSEARCH_HOSTS, ELASTICSEARCH_USERNAME, ELASTICSEARCH_PASSWORD, ELASTICSEARCH_INDEX_NAME, ELASTICSEARCH_VERSION
+func NewVectorStore() (vectorstore.VectorStore, error) {
+ storeType := VectorStoreType(GetEnvOrDefault("VECTOR_STORE_TYPE", "inmemory"))
+ return NewVectorStoreByType(storeType)
+}
+
+// NewVectorStoreByType creates a vector store based on the specified type.
+func NewVectorStoreByType(storeType VectorStoreType) (vectorstore.VectorStore, error) {
+ switch storeType {
+ case VectorStorePGVector:
+ return newPGVectorStore()
+ case VectorStoreTCVector:
+ return newTCVectorStore()
+ case VectorStoreElasticsearch:
+ return newElasticsearchStore()
+ case VectorStoreInMemory:
+ fallthrough
+ default:
+ return inmemory.New(), nil
+ }
+}
+
+func newPGVectorStore() (vectorstore.VectorStore, error) {
+ host := GetEnvOrDefault("PGVECTOR_HOST", "127.0.0.1")
+ portStr := GetEnvOrDefault("PGVECTOR_PORT", "5432")
+ port, _ := strconv.Atoi(portStr)
+ user := GetEnvOrDefault("PGVECTOR_USER", "root")
+ password := GetEnvOrDefault("PGVECTOR_PASSWORD", "")
+ database := GetEnvOrDefault("PGVECTOR_DATABASE", "vectordb")
+
+ return pgvector.New(
+ pgvector.WithHost(host),
+ pgvector.WithPort(port),
+ pgvector.WithUser(user),
+ pgvector.WithPassword(password),
+ pgvector.WithDatabase(database),
+ )
+}
+
+func newTCVectorStore() (vectorstore.VectorStore, error) {
+ url := GetEnvOrDefault("TCVECTOR_URL", "")
+ username := GetEnvOrDefault("TCVECTOR_USERNAME", "")
+ password := GetEnvOrDefault("TCVECTOR_PASSWORD", "")
+
+ if url == "" || username == "" || password == "" {
+ return nil, fmt.Errorf("TCVECTOR_URL, TCVECTOR_USERNAME, and TCVECTOR_PASSWORD are required")
+ }
+
+ return tcvector.New(
+ tcvector.WithURL(url),
+ tcvector.WithUsername(username),
+ tcvector.WithPassword(password),
+ tcvector.WithFilterAll(true),
+ )
+}
+
+func newElasticsearchStore() (vectorstore.VectorStore, error) {
+ hosts := GetEnvOrDefault("ELASTICSEARCH_HOSTS", "http://localhost:9200")
+ username := GetEnvOrDefault("ELASTICSEARCH_USERNAME", "")
+ password := GetEnvOrDefault("ELASTICSEARCH_PASSWORD", "")
+ indexName := GetEnvOrDefault("ELASTICSEARCH_INDEX_NAME", "trpc_agent_go")
+ version := GetEnvOrDefault("ELASTICSEARCH_VERSION", "v8")
+
+ hostList := strings.Split(hosts, ",")
+ return elasticsearch.New(
+ elasticsearch.WithAddresses(hostList),
+ elasticsearch.WithUsername(username),
+ elasticsearch.WithPassword(password),
+ elasticsearch.WithIndexName(indexName),
+ elasticsearch.WithVersion(version),
+ )
+}
+
+func PrintEventWithToolCalls(evt *event.Event) {
+ if evt.Error != nil {
+ log.Printf("❌ Event error: %v", evt.Error)
+ return
+ }
+
+ if len(evt.Response.Choices) == 0 {
+ return
+ }
+
+ choice := evt.Response.Choices[0]
+
+ // Print tool calls
+ if len(choice.Message.ToolCalls) > 0 {
+ fmt.Println("\n🔧 Tool Calls:")
+ for _, tc := range choice.Message.ToolCalls {
+ fmt.Printf(" - ID: %s\n", tc.ID)
+ fmt.Printf(" Function: %s\n", tc.Function.Name)
+ fmt.Printf(" Arguments: %s\n", tc.Function.Arguments)
+ }
+ }
+
+ // Print tool responses
+ if choice.Message.Role == "tool" && choice.Message.Content != "" {
+ fmt.Printf("\n📦 Tool Response (Tool Call ID: %s, Tool: %s):\n",
+ choice.Message.ToolID, choice.Message.ToolName)
+ var toolResult map[string]any
+ if err := json.Unmarshal([]byte(choice.Message.Content), &toolResult); err == nil {
+ if jsonBytes, err := json.MarshalIndent(toolResult, " ", " "); err == nil {
+ fmt.Printf("%s\n", string(jsonBytes))
+ }
+ } else {
+ fmt.Printf("%s\n", choice.Message.Content)
+ }
+ }
+}
+
+func GetEnvOrDefault(key, defaultValue string) string {
+ if value, ok := os.LookupEnv(key); ok {
+ return value
+ }
+ return defaultValue
+}
diff --git a/examples/knowledge/vectorstores/elasticsearch/README.md b/examples/knowledge/vectorstores/elasticsearch/README.md
new file mode 100644
index 000000000..c145954e6
--- /dev/null
+++ b/examples/knowledge/vectorstores/elasticsearch/README.md
@@ -0,0 +1,46 @@
+# Elasticsearch Vector Store Example
+
+Demonstrates using Elasticsearch for scalable vector search.
+
+## Prerequisites
+
+1. Start Elasticsearch:
+
+```bash
+# Docker setup (recommended)
+docker run -d \
+ --name elasticsearch \
+ -p 9200:9200 \
+ -p 9300:9300 \
+ -e "discovery.type=single-node" \
+ -e "xpack.security.enabled=false" \
+ docker.elastic.co/elasticsearch/elasticsearch:8.11.0
+```
+
+2. Set environment variables:
+
+```bash
+export ELASTICSEARCH_HOSTS=http://localhost:9200
+export ELASTICSEARCH_INDEX_NAME=trpc_agent_go
+export ELASTICSEARCH_VERSION=v9 # or v7, v8
+export OPENAI_API_KEY=your-api-key
+```
+
+## Run
+
+```bash
+go run main.go
+```
+
+## Version Support
+
+- **v7**: Elasticsearch 7.x
+- **v8**: Elasticsearch 8.0-8.7
+- **v9**: Elasticsearch 8.8+
+
+## Benefits
+
+- **High performance**: Optimized for large-scale searches
+- **Distributed**: Scalable across multiple nodes
+- **Advanced queries**: Rich filtering and aggregation
+- **Production-ready**: Battle-tested in enterprise environments
diff --git a/examples/knowledge/vectorstores/elasticsearch/main.go b/examples/knowledge/vectorstores/elasticsearch/main.go
new file mode 100644
index 000000000..f60efd901
--- /dev/null
+++ b/examples/knowledge/vectorstores/elasticsearch/main.go
@@ -0,0 +1,174 @@
+//
+// Tencent is pleased to support the open source community by making trpc-agent-go available.
+//
+// Copyright (C) 2025 Tencent. All rights reserved.
+//
+// trpc-agent-go is licensed under the Apache License Version 2.0.
+//
+//
+
+// Package main demonstrates using Elasticsearch for vector storage.
+//
+// Required environment variables:
+// - OPENAI_API_KEY: Your OpenAI API key for LLM and embeddings
+// - OPENAI_BASE_URL: (Optional) Custom OpenAI API endpoint, defaults to https://api.openai.com/v1
+// - MODEL_NAME: (Optional) Model name to use, defaults to deepseek-chat
+// - ELASTICSEARCH_HOSTS: (Optional) Elasticsearch hosts, defaults to http://localhost:9200
+// - ELASTICSEARCH_USERNAME: (Optional) Elasticsearch username
+// - ELASTICSEARCH_PASSWORD: (Optional) Elasticsearch password
+// - ELASTICSEARCH_API_KEY: (Optional) Elasticsearch API key (alternative to username/password)
+// - ELASTICSEARCH_INDEX_NAME: (Optional) Index name, defaults to trpc_agent_go
+// - ELASTICSEARCH_VERSION: (Optional) Elasticsearch version (v7, v8, v9), defaults to v9
+//
+// Example usage:
+//
+// export OPENAI_API_KEY=sk-xxxx
+// export OPENAI_BASE_URL=https://api.openai.com/v1
+// export MODEL_NAME=deepseek-chat
+// export ELASTICSEARCH_HOSTS=http://localhost:9200
+// export ELASTICSEARCH_USERNAME=elastic
+// export ELASTICSEARCH_PASSWORD=your-password
+// go run main.go
+package main
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "log"
+ "os"
+ "strings"
+ "time"
+
+ "trpc.group/trpc-go/trpc-agent-go/agent/llmagent"
+ util "trpc.group/trpc-go/trpc-agent-go/examples/knowledge"
+ "trpc.group/trpc-go/trpc-agent-go/knowledge"
+ "trpc.group/trpc-go/trpc-agent-go/knowledge/document"
+ "trpc.group/trpc-go/trpc-agent-go/knowledge/embedder/openai"
+ "trpc.group/trpc-go/trpc-agent-go/knowledge/source"
+ "trpc.group/trpc-go/trpc-agent-go/knowledge/source/file"
+ knowledgetool "trpc.group/trpc-go/trpc-agent-go/knowledge/tool"
+ "trpc.group/trpc-go/trpc-agent-go/knowledge/vectorstore/elasticsearch"
+ "trpc.group/trpc-go/trpc-agent-go/model"
+ openaimodel "trpc.group/trpc-go/trpc-agent-go/model/openai"
+ "trpc.group/trpc-go/trpc-agent-go/runner"
+ "trpc.group/trpc-go/trpc-agent-go/tool"
+)
+
+var (
+ defaultModelName = "deepseek-chat"
+)
+
+func main() {
+ ctx := context.Background()
+
+ fmt.Println("🔍 Elasticsearch Vector Store Demo")
+ fmt.Println("===================================")
+
+ // Get Elasticsearch configuration
+ hosts := getEnv("ELASTICSEARCH_HOSTS", "http://localhost:9200")
+ username := getEnv("ELASTICSEARCH_USERNAME", "")
+ password := getEnv("ELASTICSEARCH_PASSWORD", "")
+ indexName := getEnv("ELASTICSEARCH_INDEX_NAME", "trpc_agent_go")
+ version := getEnv("ELASTICSEARCH_VERSION", "v9")
+ modelName := util.GetEnvOrDefault("MODEL_NAME", defaultModelName)
+
+ fmt.Printf("📊 Elasticsearch: %s (Index: %s, Version: %s)\n", hosts, indexName, version)
+
+ // Custom document builder
+ docBuilder := func(hitSource json.RawMessage) (*document.Document, []float64, error) {
+ var source struct {
+ ID string `json:"id"`
+ Name string `json:"name"`
+ Content string `json:"content"`
+ CreatedAt time.Time `json:"created_at"`
+ UpdatedAt time.Time `json:"updated_at"`
+ Embedding []float64 `json:"embedding"`
+ Metadata map[string]any `json:"metadata"`
+ }
+ if err := json.Unmarshal(hitSource, &source); err != nil {
+ return nil, nil, err
+ }
+ doc := &document.Document{
+ ID: source.ID,
+ Name: source.Name,
+ Content: source.Content,
+ CreatedAt: source.CreatedAt,
+ UpdatedAt: source.UpdatedAt,
+ Metadata: source.Metadata,
+ }
+ return doc, source.Embedding, nil
+ }
+
+ // Create Elasticsearch vector store
+ hostList := strings.Split(hosts, ",")
+ vs, err := elasticsearch.New(
+ elasticsearch.WithAddresses(hostList),
+ elasticsearch.WithUsername(username),
+ elasticsearch.WithPassword(password),
+ elasticsearch.WithIndexName(indexName),
+ elasticsearch.WithVersion(version),
+ elasticsearch.WithMaxRetries(3),
+ elasticsearch.WithDocBuilder(docBuilder),
+ )
+ if err != nil {
+ log.Fatalf("Failed to create elasticsearch store: %v", err)
+ }
+
+ // Create file source
+ src := file.New(
+ []string{"../../exampledata/file/llm.md"},
+ file.WithName("LLM Docs"),
+ )
+
+ // Create knowledge base
+ kb := knowledge.New(
+ knowledge.WithVectorStore(vs),
+ knowledge.WithEmbedder(openai.New()),
+ knowledge.WithSources([]source.Source{src}),
+ )
+
+ fmt.Println("\n📥 Indexing knowledge into Elasticsearch...")
+ if err := kb.Load(ctx, knowledge.WithShowProgress(true)); err != nil {
+ log.Fatalf("Failed to load: %v", err)
+ }
+
+ // Create knowledge search tool
+ searchTool := knowledgetool.NewKnowledgeSearchTool(kb)
+
+ // Create agent
+ agent := llmagent.New(
+ "es-assistant",
+ llmagent.WithModel(openaimodel.New(modelName)),
+ llmagent.WithTools([]tool.Tool{searchTool}),
+ )
+
+ // Create runner
+ r := runner.NewRunner("es-chat", agent)
+ defer r.Close()
+
+ // Test query
+ fmt.Println("\n🔍 Searching Elasticsearch index...")
+ eventChan, err := r.Run(ctx, "user", "session-1",
+ model.NewUserMessage("What are transformers in machine learning?"))
+ if err != nil {
+ log.Fatalf("Run failed: %v", err)
+ }
+
+ fmt.Print("🤖 Response: ")
+ for evt := range eventChan {
+ util.PrintEventWithToolCalls(evt)
+ if evt.IsFinalResponse() && len(evt.Response.Choices) > 0 {
+ fmt.Println(evt.Response.Choices[0].Message.Content)
+ }
+ }
+
+ fmt.Println("\n✅ Data indexed in Elasticsearch!")
+}
+
+func getEnv(key, defaultValue string) string {
+ if value := os.Getenv(key); value != "" {
+ return value
+ }
+ return defaultValue
+}
diff --git a/examples/knowledge/vectorstores/postgres/README.md b/examples/knowledge/vectorstores/postgres/README.md
new file mode 100644
index 000000000..0a3f52eeb
--- /dev/null
+++ b/examples/knowledge/vectorstores/postgres/README.md
@@ -0,0 +1,45 @@
+# PostgreSQL (PGVector) Example
+
+Demonstrates persistent vector storage using PostgreSQL with pgvector extension.
+
+## Prerequisites
+
+1. Install PostgreSQL with pgvector extension:
+
+```bash
+# Docker setup (recommended)
+docker run -d \
+ --name postgres-pgvector \
+ -e POSTGRES_PASSWORD=yourpassword \
+ -e POSTGRES_DB=vectordb \
+ -p 5432:5432 \
+ pgvector/pgvector:pg16
+```
+
+2. Set environment variables:
+
+```bash
+export PGVECTOR_HOST=127.0.0.1
+export PGVECTOR_PORT=5432
+export PGVECTOR_USER=postgres
+export PGVECTOR_PASSWORD=yourpassword
+export PGVECTOR_DATABASE=vectordb
+export OPENAI_API_KEY=your-api-key
+```
+
+## Run
+
+```bash
+go run main.go
+```
+
+## Benefits
+
+- **Persistent storage**: Embeddings survive restarts
+- **Scalable**: Production-ready database
+- **Query optimization**: Built-in vector similarity search
+- **Data integrity**: ACID transactions
+
+## Reuse embeddings
+
+Run the program multiple times - it will reuse existing embeddings instead of regenerating them!
diff --git a/examples/knowledge/vectorstores/postgres/main.go b/examples/knowledge/vectorstores/postgres/main.go
new file mode 100644
index 000000000..17dc553f0
--- /dev/null
+++ b/examples/knowledge/vectorstores/postgres/main.go
@@ -0,0 +1,148 @@
+//
+// Tencent is pleased to support the open source community by making trpc-agent-go available.
+//
+// Copyright (C) 2025 Tencent. All rights reserved.
+//
+// trpc-agent-go is licensed under the Apache License Version 2.0.
+//
+//
+
+// Package main demonstrates using PostgreSQL with pgvector for persistent storage.
+//
+// Required environment variables:
+// - OPENAI_API_KEY: Your OpenAI API key for LLM and embeddings
+// - OPENAI_BASE_URL: (Optional) Custom OpenAI API endpoint, defaults to https://api.openai.com/v1
+// - MODEL_NAME: (Optional) Model name to use, defaults to deepseek-chat
+// - PGVECTOR_HOST: (Optional) PostgreSQL host, defaults to 127.0.0.1
+// - PGVECTOR_PORT: (Optional) PostgreSQL port, defaults to 5432
+// - PGVECTOR_USER: (Optional) PostgreSQL user, defaults to postgres
+// - PGVECTOR_PASSWORD: (Optional) PostgreSQL password
+// - PGVECTOR_DATABASE: (Optional) PostgreSQL database name, defaults to vectordb
+//
+// Example usage:
+//
+// export OPENAI_API_KEY=sk-xxxx
+// export OPENAI_BASE_URL=https://api.openai.com/v1
+// export MODEL_NAME=deepseek-chat
+// export PGVECTOR_HOST=127.0.0.1
+// export PGVECTOR_PORT=5432
+// export PGVECTOR_USER=postgres
+// export PGVECTOR_PASSWORD=your-password
+// export PGVECTOR_DATABASE=vectordb
+// go run main.go
+package main
+
+import (
+ "context"
+ "fmt"
+ "log"
+ "os"
+
+ "trpc.group/trpc-go/trpc-agent-go/agent/llmagent"
+ util "trpc.group/trpc-go/trpc-agent-go/examples/knowledge"
+ "trpc.group/trpc-go/trpc-agent-go/knowledge"
+ "trpc.group/trpc-go/trpc-agent-go/knowledge/embedder/openai"
+ "trpc.group/trpc-go/trpc-agent-go/knowledge/source"
+ "trpc.group/trpc-go/trpc-agent-go/knowledge/source/file"
+ knowledgetool "trpc.group/trpc-go/trpc-agent-go/knowledge/tool"
+ "trpc.group/trpc-go/trpc-agent-go/knowledge/vectorstore/pgvector"
+ "trpc.group/trpc-go/trpc-agent-go/model"
+ openaimodel "trpc.group/trpc-go/trpc-agent-go/model/openai"
+ "trpc.group/trpc-go/trpc-agent-go/runner"
+ sessioninmemory "trpc.group/trpc-go/trpc-agent-go/session/inmemory"
+ "trpc.group/trpc-go/trpc-agent-go/tool"
+)
+
+var (
+ defaultModelName = "deepseek-chat"
+)
+
+func main() {
+ ctx := context.Background()
+
+ fmt.Println("🐘 PostgreSQL (PGVector) Demo")
+ fmt.Println("==============================")
+
+ // Get PostgreSQL connection details from environment
+ host := getEnv("PGVECTOR_HOST", "127.0.0.1")
+ port := 5432
+ user := getEnv("PGVECTOR_USER", "root")
+ password := getEnv("PGVECTOR_PASSWORD", "123")
+ database := getEnv("PGVECTOR_DATABASE", "vectordb")
+ modelName := util.GetEnvOrDefault("MODEL_NAME", defaultModelName)
+
+ fmt.Printf("📊 Connecting to PostgreSQL: %s:%d/%s\n", host, port, database)
+
+ // Create PGVector store
+ vs, err := pgvector.New(
+ pgvector.WithHost(host),
+ pgvector.WithPort(port),
+ pgvector.WithUser(user),
+ pgvector.WithPassword(password),
+ pgvector.WithDatabase(database),
+ )
+ if err != nil {
+ log.Fatalf("Failed to create pgvector store: %v", err)
+ }
+
+ // Create file source
+ src := file.New(
+ []string{"../../exampledata/file/llm.md"},
+ file.WithName("LLM Docs"),
+ )
+
+ // Create knowledge base
+ kb := knowledge.New(
+ knowledge.WithVectorStore(vs),
+ knowledge.WithEmbedder(openai.New()),
+ knowledge.WithSources([]source.Source{src}),
+ )
+
+ fmt.Println("\n📥 Loading knowledge into PostgreSQL...")
+ if err := kb.Load(ctx, knowledge.WithShowProgress(true)); err != nil {
+ log.Fatalf("Failed to load: %v", err)
+ }
+
+ // Create knowledge search tool
+ searchTool := knowledgetool.NewKnowledgeSearchTool(kb)
+
+ // Create agent
+ agent := llmagent.New(
+ "pg-assistant",
+ llmagent.WithModel(openaimodel.New(modelName)),
+ llmagent.WithTools([]tool.Tool{searchTool}),
+ )
+
+ // Create runner
+ r := runner.NewRunner(
+ "pg-chat",
+ agent,
+ runner.WithSessionService(sessioninmemory.NewSessionService()),
+ )
+ defer r.Close()
+
+ // Test query
+ fmt.Println("\n🔍 Querying knowledge from PostgreSQL...")
+ eventChan, err := r.Run(ctx, "user", "session-1",
+ model.NewUserMessage("What are Large Language Models?"))
+ if err != nil {
+ log.Fatalf("Run failed: %v", err)
+ }
+
+ fmt.Print("🤖 Response: ")
+ for evt := range eventChan {
+ util.PrintEventWithToolCalls(evt)
+ if evt.IsFinalResponse() && len(evt.Response.Choices) > 0 {
+ fmt.Println(evt.Response.Choices[0].Message.Content)
+ }
+ }
+
+ fmt.Println("\n✅ Data persisted in PostgreSQL! Run again to reuse stored embeddings.")
+}
+
+func getEnv(key, defaultValue string) string {
+ if value := os.Getenv(key); value != "" {
+ return value
+ }
+ return defaultValue
+}
diff --git a/examples/knowledge/vectorstores/tcvector/README.md b/examples/knowledge/vectorstores/tcvector/README.md
new file mode 100644
index 000000000..68494ae80
--- /dev/null
+++ b/examples/knowledge/vectorstores/tcvector/README.md
@@ -0,0 +1,30 @@
+# Tencent VectorDB Example
+
+Demonstrates vector storage using Tencent VectorDB.
+
+## Prerequisites
+
+Set environment variables:
+
+```bash
+export OPENAI_BASE_URL=xxx
+export OPENAI_API_KEY=xxx
+export MODEL_NAME=xxx
+export TCVECTOR_URL=xxx
+export TCVECTOR_USERNAME=xxx
+export TCVECTOR_PASSWORD=xxx
+```
+
+## Run
+
+```bash
+go run main.go
+```
+
+## Features
+
+- **Cloud-native**: Fully managed vector database service
+- **Hybrid search**: Supports vector + text search
+- **Scalable**: Auto-scaling with high availability
+- **Persistent storage**: Embeddings survive restarts
+
diff --git a/examples/knowledge/vectorstores/tcvector/main.go b/examples/knowledge/vectorstores/tcvector/main.go
new file mode 100644
index 000000000..3513ab975
--- /dev/null
+++ b/examples/knowledge/vectorstores/tcvector/main.go
@@ -0,0 +1,145 @@
+//
+// Tencent is pleased to support the open source community by making trpc-agent-go available.
+//
+// Copyright (C) 2025 Tencent. All rights reserved.
+//
+// trpc-agent-go is licensed under the Apache License Version 2.0.
+//
+//
+
+// Package main demonstrates using Tencent VectorDB for vector storage.
+//
+// Required environment variables:
+// - OPENAI_API_KEY: Your OpenAI API key for LLM and embeddings
+// - OPENAI_BASE_URL: (Optional) Custom OpenAI API endpoint
+// - MODEL_NAME: (Optional) Model name to use, defaults to deepseek-chat
+// - TCVECTOR_URL: Tencent VectorDB URL
+// - TCVECTOR_USERNAME: Tencent VectorDB username
+// - TCVECTOR_PASSWORD: Tencent VectorDB password
+//
+// Example usage:
+//
+// export OPENAI_BASE_URL=xxx
+// export OPENAI_API_KEY=xxx
+// export MODEL_NAME=xxx
+// export TCVECTOR_URL=xxx
+// export TCVECTOR_USERNAME=xxx
+// export TCVECTOR_PASSWORD=xxx
+// go run main.go
+package main
+
+import (
+ "context"
+ "fmt"
+ "log"
+ "os"
+
+ "trpc.group/trpc-go/trpc-agent-go/agent/llmagent"
+ util "trpc.group/trpc-go/trpc-agent-go/examples/knowledge"
+ "trpc.group/trpc-go/trpc-agent-go/knowledge"
+ "trpc.group/trpc-go/trpc-agent-go/knowledge/embedder/openai"
+ "trpc.group/trpc-go/trpc-agent-go/knowledge/source"
+ "trpc.group/trpc-go/trpc-agent-go/knowledge/source/file"
+ knowledgetool "trpc.group/trpc-go/trpc-agent-go/knowledge/tool"
+ "trpc.group/trpc-go/trpc-agent-go/knowledge/vectorstore/tcvector"
+ "trpc.group/trpc-go/trpc-agent-go/model"
+ openaimodel "trpc.group/trpc-go/trpc-agent-go/model/openai"
+ "trpc.group/trpc-go/trpc-agent-go/runner"
+ sessioninmemory "trpc.group/trpc-go/trpc-agent-go/session/inmemory"
+ "trpc.group/trpc-go/trpc-agent-go/tool"
+)
+
+var (
+ defaultModelName = "deepseek-chat"
+)
+
+func main() {
+ ctx := context.Background()
+
+ fmt.Println("🔮 Tencent VectorDB Demo")
+ fmt.Println("========================")
+
+ // Get TCVector connection details from environment
+ url := getEnv("TCVECTOR_URL", "")
+ username := getEnv("TCVECTOR_USERNAME", "")
+ password := getEnv("TCVECTOR_PASSWORD", "")
+ modelName := util.GetEnvOrDefault("MODEL_NAME", defaultModelName)
+
+ if url == "" || username == "" || password == "" {
+ log.Fatal("TCVECTOR_URL, TCVECTOR_USERNAME, and TCVECTOR_PASSWORD are required")
+ }
+
+ fmt.Printf("📊 Connecting to Tencent VectorDB: %s\n", url)
+
+ // Create TCVector store
+ vs, err := tcvector.New(
+ tcvector.WithURL(url),
+ tcvector.WithUsername(username),
+ tcvector.WithPassword(password),
+ tcvector.WithFilterAll(true),
+ )
+ if err != nil {
+ log.Fatalf("Failed to create tcvector store: %v", err)
+ }
+
+ // Create file source
+ src := file.New(
+ []string{"../../exampledata/file/llm.md"},
+ file.WithName("LLM Docs"),
+ )
+
+ // Create knowledge base
+ kb := knowledge.New(
+ knowledge.WithVectorStore(vs),
+ knowledge.WithEmbedder(openai.New()),
+ knowledge.WithSources([]source.Source{src}),
+ )
+
+ fmt.Println("\n📥 Loading knowledge into Tencent VectorDB...")
+ if err := kb.Load(ctx, knowledge.WithShowProgress(true)); err != nil {
+ log.Fatalf("Failed to load: %v", err)
+ }
+
+ // Create knowledge search tool
+ searchTool := knowledgetool.NewKnowledgeSearchTool(kb)
+
+ // Create agent
+ agent := llmagent.New(
+ "tcvector-assistant",
+ llmagent.WithModel(openaimodel.New(modelName)),
+ llmagent.WithTools([]tool.Tool{searchTool}),
+ )
+
+ // Create runner
+ r := runner.NewRunner(
+ "tcvector-chat",
+ agent,
+ runner.WithSessionService(sessioninmemory.NewSessionService()),
+ )
+ defer r.Close()
+
+ // Test query
+ fmt.Println("\n🔍 Querying knowledge from Tencent VectorDB...")
+ eventChan, err := r.Run(ctx, "user", "session-1",
+ model.NewUserMessage("What are Large Language Models?"))
+ if err != nil {
+ log.Fatalf("Run failed: %v", err)
+ }
+
+ fmt.Print("🤖 Response: ")
+ for evt := range eventChan {
+ util.PrintEventWithToolCalls(evt)
+ if evt.IsFinalResponse() && len(evt.Response.Choices) > 0 {
+ fmt.Println(evt.Response.Choices[0].Message.Content)
+ }
+ }
+
+ fmt.Println("\n✅ Data persisted in Tencent VectorDB!")
+}
+
+func getEnv(key, defaultValue string) string {
+ if value := os.Getenv(key); value != "" {
+ return value
+ }
+ return defaultValue
+}
diff --git a/knowledge/knowledge.go b/knowledge/knowledge.go
index 32f4413d6..03ce85174 100644
--- a/knowledge/knowledge.go
+++ b/knowledge/knowledge.go
@@ -16,6 +16,7 @@ import (
"trpc.group/trpc-go/trpc-agent-go/knowledge/document"
"trpc.group/trpc-go/trpc-agent-go/knowledge/query"
"trpc.group/trpc-go/trpc-agent-go/knowledge/searchfilter"
+ "trpc.group/trpc-go/trpc-agent-go/knowledge/source"
)
// Knowledge is the main interface for knowledge management operations.
@@ -91,3 +92,9 @@ type SearchFilter struct {
// It is compatible with all storage engines.
FilterCondition *searchfilter.UniversalFilterCondition
}
+
+// MetadataFieldPrefix is re-exported from source package for convenience.
+// Fields with this prefix (e.g., "metadata.category") are treated as metadata fields
+// and will be processed accordingly by each vector store implementation.
+// Fields without this prefix are treated as system fields or custom top-level fields.
+const MetadataFieldPrefix = source.MetadataFieldPrefix
diff --git a/knowledge/source/source.go b/knowledge/source/source.go
index f9fd7f20e..961de5bce 100644
--- a/knowledge/source/source.go
+++ b/knowledge/source/source.go
@@ -73,7 +73,12 @@ type Source interface {
GetMetadata() map[string]any
}
+// MetadataFieldPrefix is the prefix for metadata fields in filter conditions.
+// Fields with this prefix are treated as metadata fields and will be processed accordingly.
+const MetadataFieldPrefix = "metadata."
+
// GetAllMetadata returns all metadata collected from sources with deduplication.
+// Keys are prefixed with "metadata." for use in filter conditions.
func GetAllMetadata(sources []Source) map[string][]any {
// Use temporary map for deduplication
tempMetadataMap := make(map[string]map[string]struct{})
@@ -83,17 +88,20 @@ func GetAllMetadata(sources []Source) map[string][]any {
for _, src := range sources {
metadata := src.GetMetadata()
for key, value := range metadata {
+ // Add metadata prefix to key
+ prefixedKey := MetadataFieldPrefix + key
+
// Initialize key in temporary map
- if _, exists := tempMetadataMap[key]; !exists {
- tempMetadataMap[key] = make(map[string]struct{})
- allMetadata[key] = make([]any, 0)
+ if _, exists := tempMetadataMap[prefixedKey]; !exists {
+ tempMetadataMap[prefixedKey] = make(map[string]struct{})
+ allMetadata[prefixedKey] = make([]any, 0)
}
// Create a unique key that includes type information to avoid conflicts
valueKey := fmt.Sprintf("%T:%v", value, value)
- if _, exists := tempMetadataMap[key][valueKey]; !exists {
- allMetadata[key] = append(allMetadata[key], value)
- tempMetadataMap[key][valueKey] = struct{}{}
+ if _, exists := tempMetadataMap[prefixedKey][valueKey]; !exists {
+ allMetadata[prefixedKey] = append(allMetadata[prefixedKey], value)
+ tempMetadataMap[prefixedKey][valueKey] = struct{}{}
}
}
}
@@ -101,13 +109,15 @@ func GetAllMetadata(sources []Source) map[string][]any {
}
// GetAllMetadataWithoutValues returns all metadata keys with their string values collected from sources with deduplication.
+// Keys are prefixed with "metadata." for use in filter conditions.
func GetAllMetadataWithoutValues(sources []Source) map[string][]any {
result := make(map[string][]any)
for _, src := range sources {
metadata := src.GetMetadata()
for key := range metadata {
- if _, exists := result[key]; !exists {
- result[key] = []any{}
+ prefixedKey := MetadataFieldPrefix + key
+ if _, exists := result[prefixedKey]; !exists {
+ result[prefixedKey] = []any{}
}
}
}
@@ -115,6 +125,7 @@ func GetAllMetadataWithoutValues(sources []Source) map[string][]any {
}
// GetAllMetadataKeys returns all metadata keys collected from sources with deduplication.
+// Keys are prefixed with "metadata." for use in filter conditions.
func GetAllMetadataKeys(sources []Source) []string {
allMetadata := GetAllMetadataWithoutValues(sources)
result := make([]string, 0)
diff --git a/knowledge/source/source_test.go b/knowledge/source/source_test.go
index c852f67e2..e073d068f 100644
--- a/knowledge/source/source_test.go
+++ b/knowledge/source/source_test.go
@@ -65,8 +65,8 @@ func TestGetAllMetadata(t *testing.T) {
},
},
expected: map[string][]any{
- "category": {"docs"},
- "version": {"1.0"},
+ "metadata.category": {"docs"},
+ "metadata.version": {"1.0"},
},
},
{
@@ -90,8 +90,8 @@ func TestGetAllMetadata(t *testing.T) {
},
},
expected: map[string][]any{
- "category": {"docs", "api"},
- "version": {"1.0", "2.0"},
+ "metadata.category": {"docs", "api"},
+ "metadata.version": {"1.0", "2.0"},
},
},
{
@@ -115,8 +115,8 @@ func TestGetAllMetadata(t *testing.T) {
},
},
expected: map[string][]any{
- "category": {"docs"}, // should be deduplicated
- "version": {"1.0", "2.0"},
+ "metadata.category": {"docs"}, // should be deduplicated
+ "metadata.version": {"1.0", "2.0"},
},
},
{
@@ -138,7 +138,7 @@ func TestGetAllMetadata(t *testing.T) {
},
},
expected: map[string][]any{
- "count": {5, "5"},
+ "metadata.count": {5, "5"},
},
},
}
@@ -195,8 +195,8 @@ func TestGetAllMetadataWithoutValues(t *testing.T) {
},
},
expected: map[string][]any{
- "category": {},
- "version": {},
+ "metadata.category": {},
+ "metadata.version": {},
},
},
{
@@ -220,9 +220,9 @@ func TestGetAllMetadataWithoutValues(t *testing.T) {
},
},
expected: map[string][]any{
- "category": {},
- "version": {},
- "author": {},
+ "metadata.category": {},
+ "metadata.version": {},
+ "metadata.author": {},
},
},
}
@@ -265,7 +265,7 @@ func TestGetAllMetadataKeys(t *testing.T) {
},
},
},
- expected: []string{"category", "version"},
+ expected: []string{"metadata.category", "metadata.version"},
},
{
name: "multiple sources with overlapping keys",
@@ -287,7 +287,7 @@ func TestGetAllMetadataKeys(t *testing.T) {
},
},
},
- expected: []string{"category", "version", "author"},
+ expected: []string{"metadata.category", "metadata.version", "metadata.author"},
},
}
diff --git a/knowledge/tool/searchtool.go b/knowledge/tool/searchtool.go
index ef2cdac11..44882cb5a 100644
--- a/knowledge/tool/searchtool.go
+++ b/knowledge/tool/searchtool.go
@@ -365,21 +365,19 @@ func generateAgenticFilterPrompt(agenticFilterInfo map[string][]any) string {
var b strings.Builder
- fmt.Fprintf(&b, `You are a helpful assistant that can search for relevant information in the knowledge base. Available metadata filters: %s.
+ fmt.Fprintf(&b, `You are a helpful assistant that can search for relevant information in the knowledge base. Available filters: %s.
Filter Usage:
- Query: Can be empty when using only metadata filters
- Filter: Use "filter" field with standard operators (lowercase): eq, ne, gt, gte, lt, lte, in, not in, like, not like, between, and, or
+- Field names: Use the EXACT field names from available filters. Some fields have "metadata." prefix (for document metadata), some don't (for system/custom fields). Always use the field name exactly as shown.
Filter Examples (use double quotes for JSON):
-- Single: {"field": "category", "operator": "eq", "value": "documentation"}
-- OR: {"operator": "or", "value": [{"field": "type", "operator": "eq", "value": "golang"}, {"field": "type", "operator": "eq", "value": "llm"}]}
-- AND: {"operator": "and", "value": [{"field": "category", "operator": "eq", "value": "doc"}, {"field": "topic", "operator": "eq", "value": "programming"}]}
-- IN: {"field": "type", "operator": "in", "value": ["golang", "llm", "wiki"]}
-- NOT IN: {"field": "status", "operator": "not in", "value": ["archived", "deleted"]}
-- LIKE: {"field": "title", "operator": "like", "value": "%%tutorial%%"}
-- BETWEEN: {"field": "score", "operator": "between", "value": [0.5, 0.9]}
-- Nested: {"operator": "and", "value": [{"field": "category", "operator": "eq", "value": "doc"}, {"operator": "or", "value": [{"field": "topic", "operator": "eq", "value": "programming"}, {"field": "topic", "operator": "eq", "value": "ml"}]}]}
+- Single: {"field": "metadata.category", "operator": "eq", "value": "documentation"}
+- OR: {"operator": "or", "value": [{"field": "metadata.type", "operator": "eq", "value": "golang"}, {"field": "metadata.type", "operator": "eq", "value": "llm"}]}
+- AND: {"operator": "and", "value": [{"field": "metadata.category", "operator": "eq", "value": "doc"}, {"field": "metadata.topic", "operator": "eq", "value": "programming"}]}
+- IN: {"field": "metadata.type", "operator": "in", "value": ["golang", "llm", "wiki"]}
+- System field (no prefix): {"field": "id", "operator": "eq", "value": "doc-123"}
Note: For logical operators (and/or), use "value" field to specify an array of sub-conditions.
diff --git a/knowledge/tool/searchtool_test.go b/knowledge/tool/searchtool_test.go
index e6894be1a..84f406a96 100644
--- a/knowledge/tool/searchtool_test.go
+++ b/knowledge/tool/searchtool_test.go
@@ -308,7 +308,7 @@ func TestAgenticFilterSearchTool(t *testing.T) {
require.NotEmpty(t, decl.Name)
require.Equal(t, "knowledge_search_with_agentic_filter", decl.Name)
require.NotEmpty(t, decl.Description)
- require.Contains(t, decl.Description, "Available metadata filters")
+ require.Contains(t, decl.Description, "Available filters:")
require.Contains(t, decl.Description, "category")
require.Contains(t, decl.Description, "protocol")
require.Contains(t, decl.Description, "level")
@@ -483,7 +483,7 @@ func TestGenerateAgenticFilterPrompt(t *testing.T) {
prompt := generateAgenticFilterPrompt(filterInfo)
// Check for new prompt structure
- require.Contains(t, prompt, "Available metadata filters")
+ require.Contains(t, prompt, "Available filters:")
require.Contains(t, prompt, "category")
require.Contains(t, prompt, "protocol")
require.Contains(t, prompt, "empty")
diff --git a/knowledge/vectorstore/elasticsearch/condition_converter.go b/knowledge/vectorstore/elasticsearch/condition_converter.go
index 81d7dbb40..e82a33fdf 100644
--- a/knowledge/vectorstore/elasticsearch/condition_converter.go
+++ b/knowledge/vectorstore/elasticsearch/condition_converter.go
@@ -18,11 +18,23 @@ import (
"github.com/elastic/go-elasticsearch/v9/typedapi/types"
"trpc.group/trpc-go/trpc-agent-go/knowledge/searchfilter"
+ "trpc.group/trpc-go/trpc-agent-go/knowledge/source"
"trpc.group/trpc-go/trpc-agent-go/log"
)
// esConverter converts a filter condition to an Elasticsearch query.
-type esConverter struct{}
+type esConverter struct {
+ metadataFieldName string
+}
+
+// convertFieldName converts metadata.xxx fields to ES field path.
+func (c *esConverter) convertFieldName(field string) string {
+ if strings.HasPrefix(field, source.MetadataFieldPrefix) {
+ actualField := strings.TrimPrefix(field, source.MetadataFieldPrefix)
+ return fmt.Sprintf("%s.%s", c.metadataFieldName, actualField)
+ }
+ return field
+}
// Convert converts a filter condition to an Elasticsearch query filter.
func (c *esConverter) Convert(cond *searchfilter.UniversalFilterCondition) (types.QueryVariant, error) {
@@ -114,9 +126,10 @@ func (c *esConverter) buildComparisonCondition(cond *searchfilter.UniversalFilte
}
func (c *esConverter) convertEqual(cond *searchfilter.UniversalFilterCondition) (types.QueryVariant, error) {
+ fieldName := c.convertFieldName(cond.Field)
return &types.Query{
Term: map[string]types.TermQuery{
- cond.Field: {
+ fieldName: {
Value: cond.Value,
},
},
@@ -124,12 +137,13 @@ func (c *esConverter) convertEqual(cond *searchfilter.UniversalFilterCondition)
}
func (c *esConverter) convertNotEqual(cond *searchfilter.UniversalFilterCondition) (types.QueryVariant, error) {
+ fieldName := c.convertFieldName(cond.Field)
return &types.Query{
Bool: &types.BoolQuery{
MustNot: []types.Query{
{
Term: map[string]types.TermQuery{
- cond.Field: {
+ fieldName: {
Value: cond.Value,
},
},
@@ -140,9 +154,10 @@ func (c *esConverter) convertNotEqual(cond *searchfilter.UniversalFilterConditio
}
func (c *esConverter) convertRange(cond *searchfilter.UniversalFilterCondition) (types.QueryVariant, error) {
+ fieldName := c.convertFieldName(cond.Field)
return &types.Query{
Range: map[string]types.RangeQuery{
- cond.Field: map[string]any{
+ fieldName: map[string]any{
cond.Operator: cond.Value,
},
},
@@ -158,9 +173,10 @@ func (c *esConverter) buildBetweenCondition(cond *searchfilter.UniversalFilterCo
return nil, fmt.Errorf("between operator value must be a slice with two elements: %v", cond.Value)
}
+ fieldName := c.convertFieldName(cond.Field)
return &types.Query{
Range: map[string]types.RangeQuery{
- cond.Field: map[string]any{
+ fieldName: map[string]any{
"gte": value.Index(0).Interface(),
"lte": value.Index(1).Interface(),
},
@@ -177,10 +193,11 @@ func (c *esConverter) buildInCondition(cond *searchfilter.UniversalFilterConditi
return nil, fmt.Errorf("in operator value must be a slice with at least one element: %v", cond.Value)
}
+ fieldName := c.convertFieldName(cond.Field)
termsQuery := types.Query{
Terms: &types.TermsQuery{
TermsQuery: map[string]types.TermsQueryField{
- cond.Field: cond.Value,
+ fieldName: cond.Value,
},
},
}
@@ -205,12 +222,13 @@ func (c *esConverter) buildLikeCondition(cond *searchfilter.UniversalFilterCondi
return nil, fmt.Errorf("like operator value must be a string: %v", cond.Value)
}
+ fieldName := c.convertFieldName(cond.Field)
wildcardPattern := strings.ReplaceAll(valueStr, "%", "*")
wildcardPattern = strings.ReplaceAll(wildcardPattern, "_", "?")
wildcardQuery := types.Query{
Wildcard: map[string]types.WildcardQuery{
- cond.Field: {
+ fieldName: {
Value: &wildcardPattern,
},
},
diff --git a/knowledge/vectorstore/elasticsearch/elasticsearch.go b/knowledge/vectorstore/elasticsearch/elasticsearch.go
index 61d078a1c..343812a56 100644
--- a/knowledge/vectorstore/elasticsearch/elasticsearch.go
+++ b/knowledge/vectorstore/elasticsearch/elasticsearch.go
@@ -99,7 +99,7 @@ func New(opts ...Option) (*VectorStore, error) {
vs := &VectorStore{
client: client,
option: option,
- filterConverter: &esConverter{},
+ filterConverter: &esConverter{metadataFieldName: option.metadataFieldName},
}
// Ensure index exists with proper mapping.
diff --git a/knowledge/vectorstore/inmemory/condition_converter.go b/knowledge/vectorstore/inmemory/condition_converter.go
index b273aa604..a0ee3ea78 100644
--- a/knowledge/vectorstore/inmemory/condition_converter.go
+++ b/knowledge/vectorstore/inmemory/condition_converter.go
@@ -20,6 +20,7 @@ import (
"trpc.group/trpc-go/trpc-agent-go/knowledge/document"
"trpc.group/trpc-go/trpc-agent-go/knowledge/searchfilter"
+ "trpc.group/trpc-go/trpc-agent-go/knowledge/source"
"trpc.group/trpc-go/trpc-agent-go/log"
)
@@ -413,8 +414,8 @@ func isValidField(field string) bool {
return true
}
- // metadata fields are prefixed with "metadata."
- if strings.HasPrefix(field, "metadata.") {
+ // metadata fields are prefixed with source.MetadataFieldPrefix
+ if strings.HasPrefix(field, source.MetadataFieldPrefix) {
return true
}
return false
@@ -437,12 +438,12 @@ func fieldValue(doc *document.Document, field string) (any, bool) {
case updatedAtField:
return doc.UpdatedAt, true
default:
- if !strings.HasPrefix(field, "metadata.") {
+ if !strings.HasPrefix(field, source.MetadataFieldPrefix) {
return nil, false
}
// metadata fields
- elementKey := strings.TrimPrefix(field, "metadata.")
+ elementKey := strings.TrimPrefix(field, source.MetadataFieldPrefix)
if val, ok := doc.Metadata[elementKey]; ok {
return val, true
}
diff --git a/knowledge/vectorstore/pgvector/condition_converter.go b/knowledge/vectorstore/pgvector/condition_converter.go
index cd7a6ed10..971ee271a 100644
--- a/knowledge/vectorstore/pgvector/condition_converter.go
+++ b/knowledge/vectorstore/pgvector/condition_converter.go
@@ -16,6 +16,7 @@ import (
"strings"
"trpc.group/trpc-go/trpc-agent-go/knowledge/searchfilter"
+ "trpc.group/trpc-go/trpc-agent-go/knowledge/source"
"trpc.group/trpc-go/trpc-agent-go/log"
)
@@ -34,7 +35,9 @@ type condConvertResult struct {
}
// pgVectorConverter converts a filter condition to a postgres vector query.
-type pgVectorConverter struct{}
+type pgVectorConverter struct {
+ metadataFieldName string
+}
// Convert converts a filter condition to a postgres vector query filter.
func (c *pgVectorConverter) Convert(cond *searchfilter.UniversalFilterCondition) (*condConvertResult, error) {
@@ -48,6 +51,15 @@ func (c *pgVectorConverter) Convert(cond *searchfilter.UniversalFilterCondition)
return c.convertCondition(cond)
}
+// convertFieldName converts metadata.xxx fields to JSONB syntax.
+func (c *pgVectorConverter) convertFieldName(field string) string {
+ if strings.HasPrefix(field, source.MetadataFieldPrefix) {
+ actualField := strings.TrimPrefix(field, source.MetadataFieldPrefix)
+ return fmt.Sprintf("%s->>'%s'", c.metadataFieldName, actualField)
+ }
+ return field
+}
+
func (c *pgVectorConverter) convertCondition(cond *searchfilter.UniversalFilterCondition) (*condConvertResult, error) {
if cond == nil {
return nil, fmt.Errorf("nil condition")
@@ -81,6 +93,7 @@ func (c *pgVectorConverter) buildInCondition(cond *searchfilter.UniversalFilterC
return nil, fmt.Errorf("in operator value must be a slice with at least one value: %v", cond.Value)
}
+ fieldName := c.convertFieldName(cond.Field)
itemNum := value.Len()
condResult := condConvertResult{args: make([]any, 0, itemNum)}
args := make([]string, 0, itemNum)
@@ -88,7 +101,7 @@ func (c *pgVectorConverter) buildInCondition(cond *searchfilter.UniversalFilterC
condResult.args = append(condResult.args, value.Index(i).Interface())
args = append(args, "$%d")
}
- condResult.cond = fmt.Sprintf(`%s %s (%s)`, cond.Field, strings.ToUpper(cond.Operator), strings.Join(args, ", "))
+ condResult.cond = fmt.Sprintf(`%s %s (%s)`, fieldName, strings.ToUpper(cond.Operator), strings.Join(args, ", "))
return &condResult, nil
}
@@ -132,8 +145,9 @@ func (c *pgVectorConverter) buildComparisonCondition(cond *searchfilter.Universa
return nil, fmt.Errorf("field is empty")
}
+ fieldName := c.convertFieldName(cond.Field)
return &condConvertResult{
- cond: fmt.Sprintf(`%s %s `, cond.Field, operator) + "$%d",
+ cond: fmt.Sprintf(`%s %s `, fieldName, operator) + "$%d",
args: []any{cond.Value},
}, nil
}
@@ -146,8 +160,9 @@ func (c *pgVectorConverter) buildLikeCondition(cond *searchfilter.UniversalFilte
return nil, fmt.Errorf("like operator value must be a string: %v", cond.Value)
}
+ fieldName := c.convertFieldName(cond.Field)
return &condConvertResult{
- cond: fmt.Sprintf(`%s %s `, cond.Field, strings.ToUpper(cond.Operator)) + "$%d",
+ cond: fmt.Sprintf(`%s %s `, fieldName, strings.ToUpper(cond.Operator)) + "$%d",
args: []any{cond.Value},
}, nil
}
@@ -161,8 +176,9 @@ func (c *pgVectorConverter) buildBetweenCondition(cond *searchfilter.UniversalFi
return nil, fmt.Errorf("between operator value must be a slice with two elements: %v", cond.Value)
}
+ fieldName := c.convertFieldName(cond.Field)
return &condConvertResult{
- cond: cond.Field + " >= $%d AND " + cond.Field + " <= $%d",
+ cond: fieldName + " >= $%d AND " + fieldName + " <= $%d",
args: []any{value.Index(0).Interface(), value.Index(1).Interface()},
}, nil
}
diff --git a/knowledge/vectorstore/pgvector/pgvector.go b/knowledge/vectorstore/pgvector/pgvector.go
index e2a3002e8..eef46c1bd 100644
--- a/knowledge/vectorstore/pgvector/pgvector.go
+++ b/knowledge/vectorstore/pgvector/pgvector.go
@@ -136,7 +136,7 @@ func New(opts ...Option) (*VectorStore, error) {
vs := &VectorStore{
client: client,
option: option,
- filterConverter: &pgVectorConverter{},
+ filterConverter: &pgVectorConverter{metadataFieldName: option.metadataFieldName},
}
if err := vs.initDB(context.Background()); err != nil {
@@ -783,6 +783,7 @@ func (vs *VectorStore) buildQueryFilter(qb queryFilterBuilder, cond *vectorstore
if cond.FilterCondition == nil {
return nil
}
+
filter, err := vs.filterConverter.Convert(cond.FilterCondition)
if err != nil {
return err