feat: tencentcloud hunyuan embedding

components/embedding/tencentcloud/README.md

@@ -0,0 +1,94 @@
+# Tencent Cloud Hunyuan Embedding
+
+English | [简体中文](README_zh.md)
+
+A Tencent Cloud Hunyuan embedding implementation for [Eino](https://github.com/cloudwego/eino) that implements the `Embedder` interface. This enables seamless integration with Eino's embedding system for text embedding capabilities.
+
+## Features
+
+- Implements `github.com/cloudwego/eino/components/embedding.Embedder`
+- Easy integration with Eino's rag workflow
+- Built-in token usage tracking
+- Automatic batch processing for large text arrays
+- Built-in callback support
+
+## Installation
+
+```bash
+go get github.com/cloudwego/eino-ext/components/embedding/tencentcloud
+```
+
+## Quick Start
+
+```go
+package main
+
+import (
+    "context"
+    "fmt"
+    "os"
+
+    "github.com/cloudwego/eino-ext/components/embedding/tencentcloud"
+)
+
+func main() {
+    ctx := context.Background()
+
+    // Create embedder config
+    cfg := &tencentcloud.EmbeddingConfig{
+        SecretID:  os.Getenv("TENCENTCLOUD_SECRET_ID"),
+        SecretKey: os.Getenv("TENCENTCLOUD_SECRET_KEY"),
+        Region:    "ap-guangzhou",
+    }
+
+    // Create the embedder
+    embedder, err := tencentcloud.NewEmbedder(ctx, cfg)
+    if err != nil {
+        panic(err)
+    }
+
+    // Get embeddings for texts
+    embeddings, err := embedder.EmbedStrings(ctx, []string{"hello world", "bye world"})
+    if err != nil {
+        panic(err)
+    }
+
+    fmt.Printf("Embeddings: %v\n", embeddings)
+}
+```
+
+## Configuration
+
+The embedder can be configured using the `EmbeddingConfig` struct:
+
+```go
+type EmbeddingConfig struct {
+    SecretID  string // Tencent Cloud Secret ID
+    SecretKey string // Tencent Cloud Secret Key
+    Region    string // Tencent Cloud Region (e.g. "ap-hongkong")
+}
+```
+
+## Features Details
+
+### Automatic Batch Processing
+
+The embedder automatically handles batch processing for large text arrays. According to Tencent Cloud's API limitations, each request can process up to 200 texts. The embedder will automatically split larger arrays into appropriate batches.
+
+### Token Usage Tracking
+
+The embedder tracks token usage through Eino's callback system. Token usage information includes:
+- Prompt tokens
+- Total tokens
+
+### Callbacks Support
+
+The embedder fully supports Eino's callback system, enabling:
+- Error tracking
+- Start/End event monitoring
+- Token usage statistics
+
+## For More Details
+
+- [Tencent Cloud Hunyuan API Documentation](https://cloud.tencent.com/document/product/1729/102832)
+- [Eino Documentation](https://github.com/cloudwego/eino)

components/embedding/tencentcloud/README_zh.md

@@ -0,0 +1,94 @@
+# 腾讯云混元 Embedding
+
+[English](README.md) | 简体中文
+
+这是一个为 [Eino](https://github.com/cloudwego/eino) 实现的腾讯云混元 Embedding 组件，实现了 `Embedder` 接口。它可以无缝集成到 Eino 的 embedding 系统中，提供文本向量化能力。
+
+## 特性
+
+- 实现了 `github.com/cloudwego/eino/components/embedding.Embedder` 接口
+- 易于集成到 Eino 的 rag 工作流中
+- 内置 token 使用量追踪
+- 自动处理大规模文本数组的批处理
+- 内置回调支持
+
+## 安装
+
+```bash
+go get github.com/cloudwego/eino-ext/components/embedding/tencentcloud
+```
+
+## 快速开始
+
+```go
+package main
+
+import (
+    "context"
+    "fmt"
+    "os"
+
+    "github.com/cloudwego/eino-ext/components/embedding/tencentcloud"
+)
+
+func main() {
+    ctx := context.Background()
+
+    // 创建 embedder 配置
+    cfg := &tencentcloud.EmbeddingConfig{
+        SecretID:  os.Getenv("TENCENTCLOUD_SECRET_ID"),
+        SecretKey: os.Getenv("TENCENTCLOUD_SECRET_KEY"),
+        Region:    "ap-guangzhou",
+    }
+
+    // 创建 embedder
+    embedder, err := tencentcloud.NewEmbedder(ctx, cfg)
+    if err != nil {
+        panic(err)
+    }
+
+    // 获取文本的向量表示
+    embeddings, err := embedder.EmbedStrings(ctx, []string{"hello world", "bye world"})
+    if err != nil {
+        panic(err)
+    }
+
+    fmt.Printf("Embeddings: %v\n", embeddings)
+}
+```
+
+## 配置说明
+
+embedder 可以通过 `EmbeddingConfig` 结构体进行配置：
+
+```go
+type EmbeddingConfig struct {
+    SecretID  string // 腾讯云 Secret ID
+    SecretKey string // 腾讯云 Secret Key
+    Region    string // 腾讯云地域（如 "ap-guangzhou"）
+}
+```
+
+## 功能详情
+
+### 自动批处理
+
+embedder 会自动处理大规模文本数组的批处理。根据腾讯云 API 的限制，每个请求最多可以处理 200 个文本。embedder 会自动将较大的数组分割成适当的批次进行处理。
+
+### Token 使用量追踪
+
+embedder 通过 Eino 的回调系统追踪 token 使用量。token 使用信息包括：
+- 输入 token 数量
+- 总 token 数量
+
+### 回调支持
+
+embedder 完全支持 Eino 的回调系统，支持：
+- 错误追踪
+- 开始/结束事件监控
+- Token 使用统计
+
+## 更多信息
+
+- [腾讯云混元 API 文档](https://cloud.tencent.com/document/product/1729/102832)
+- [Eino 文档](https://github.com/cloudwego/eino) 

components/embedding/tencentcloud/embedding.go

@@ -0,0 +1,131 @@
+/*
+ * Copyright 2024 CloudWeGo Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package tencentcloud
+
+import (
+	"context"
+
+	"github.com/cloudwego/eino/callbacks"
+	"github.com/cloudwego/eino/components/embedding"
+
+	"github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common"
+	"github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/profile"
+	hunyuan "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/hunyuan/v20230901"
+)
+
+const defaultModel = "hunyuan-embedding"
+
+type EmbeddingConfig struct {
+	SecretID  string
+	SecretKey string
+	Region    string
+}
+
+var _ embedding.Embedder = (*Embedder)(nil)
+
+// Embedder is a Tencent Cloud embedding client
+type Embedder struct {
+	client *hunyuan.Client
+}
+
+// NewEmbedder creates a new Tencent Cloud embedding client
+func NewEmbedder(ctx context.Context, config *EmbeddingConfig) (*Embedder, error) {
+	credential := common.NewCredential(
+		config.SecretID,
+		config.SecretKey,
+	)
+	profile := profile.NewClientProfile()
+
+	client, err := hunyuan.NewClient(credential, config.Region, profile)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Embedder{
+		client: client,
+	}, nil
+}
+
+func (e *Embedder) EmbedStrings(ctx context.Context, texts []string, opts ...embedding.Option) (
+	embeddings [][]float64, err error,
+) {
+	defer func() {
+		if err != nil {
+			callbacks.OnError(ctx, err)
+		}
+	}()
+
+	conf := &embedding.Config{
+		Model: defaultModel, // hunyuan embedding does not specify model
+	}
+
+	ctx = callbacks.OnStart(ctx, &embedding.CallbackInput{
+		Texts:  texts,
+		Config: conf,
+	})
+
+	// NOTE: len of req.InputList must less equal than 200, so we need to split texts into batches
+	// reference: https://pkg.go.dev/github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/hunyuan@v1.0.1093/v20230901#GetEmbeddingRequest.InputList
+	batchSize := 200
+
+	req := hunyuan.NewGetEmbeddingRequest()
+	req.SetContext(ctx)
+
+	promptTokens, totalTokens := 0, 0
+	embeddings = make([][]float64, len(texts))
+	for l := 0; l < len(texts); l += batchSize {
+		r := min(l+batchSize, len(texts))
+
+		req.InputList = common.StringPtrs(texts[l:r])
+		rsp, err := e.client.GetEmbedding(req)
+		if err != nil {
+			return nil, err
+		}
+
+		for idx, d := range rsp.Response.Data {
+			embeddings[l+idx] = make([]float64, len(d.Embedding))
+			for i, emb := range d.Embedding { // *float64 -> float64
+				embeddings[l+idx][i] = *emb
+			}
+		}
+
+		promptTokens += int(*rsp.Response.Usage.PromptTokens)
+		totalTokens += int(*rsp.Response.Usage.TotalTokens)
+	}
+
+	callbacks.OnEnd(ctx, &embedding.CallbackOutput{
+		Embeddings: embeddings,
+		Config:     conf,
+		TokenUsage: &embedding.TokenUsage{
+			PromptTokens:     promptTokens,
+			CompletionTokens: 0, // hunyuan embedding does not has completion tokens
+			TotalTokens:      totalTokens,
+		},
+	})
+
+	return embeddings, nil
+}
+
+const typ = "TencentCloud"
+
+func (e *Embedder) GetType() string {
+	return typ
+}
+
+func (e *Embedder) IsCallbacksEnabled() bool {
+	return true
+}