diff --git a/backend/api/api.go b/backend/api/api.go index 53f0c649d..7bbd0acc7 100644 --- a/backend/api/api.go +++ b/backend/api/api.go @@ -7,6 +7,7 @@ package api import ( "context" + "github.com/coze-dev/coze-loop/backend/modules/observability/infra/storage" "github.com/cloudwego/hertz/pkg/app/server" "github.com/cloudwego/hertz/pkg/app/server/binding" @@ -123,6 +124,7 @@ func Init( lodataset.NewLocalDatasetService(dataHandler.IDatasetApplication), cmdable, persistentCmdable, + storage.NewTraceStorageProvider(), loexpt.NewLocalExperimentService(evaluationHandler.IExperimentApplication), processor.TaskProcessor{}, 0, diff --git a/backend/api/handler/coze/loop/apis/wire.go b/backend/api/handler/coze/loop/apis/wire.go index cc9d73859..03609adca 100644 --- a/backend/api/handler/coze/loop/apis/wire.go +++ b/backend/api/handler/coze/loop/apis/wire.go @@ -45,6 +45,7 @@ import ( foundationapp "github.com/coze-dev/coze-loop/backend/modules/foundation/application" llmapp "github.com/coze-dev/coze-loop/backend/modules/llm/application" obapp "github.com/coze-dev/coze-loop/backend/modules/observability/application" + "github.com/coze-dev/coze-loop/backend/modules/observability/domain/component/storage" promptapp "github.com/coze-dev/coze-loop/backend/modules/prompt/application" "github.com/coze-dev/coze-loop/backend/pkg/conf" ) @@ -216,6 +217,7 @@ func InitObservabilityHandler( datasetClient datasetservice.Client, redis redis.Cmdable, persistentCmdable redis.PersistentCmdable, + storageProvider storage.IStorageProvider, experimentClient experimentservice.Client, taskProcessor task_processor.TaskProcessor, aid int32, diff --git a/backend/api/handler/coze/loop/apis/wire_gen.go b/backend/api/handler/coze/loop/apis/wire_gen.go index ab2d3667c..19169865b 100644 --- a/backend/api/handler/coze/loop/apis/wire_gen.go +++ b/backend/api/handler/coze/loop/apis/wire_gen.go @@ -8,7 +8,6 @@ package apis import ( "context" - "github.com/cloudwego/kitex/pkg/endpoint" "github.com/coze-dev/coze-loop/backend/infra/ck" "github.com/coze-dev/coze-loop/backend/infra/db" @@ -41,6 +40,7 @@ import ( "github.com/coze-dev/coze-loop/backend/modules/foundation/application" application3 "github.com/coze-dev/coze-loop/backend/modules/llm/application" application6 "github.com/coze-dev/coze-loop/backend/modules/observability/application" + "github.com/coze-dev/coze-loop/backend/modules/observability/domain/component/storage" "github.com/coze-dev/coze-loop/backend/modules/observability/domain/task/service/taskexe/processor" application2 "github.com/coze-dev/coze-loop/backend/modules/prompt/application" "github.com/coze-dev/coze-loop/backend/pkg/conf" @@ -155,12 +155,12 @@ func InitDataHandler(ctx context.Context, idgen2 idgen.IIDGenerator, db2 db.Prov return dataHandler, nil } -func InitObservabilityHandler(ctx context.Context, db2 db.Provider, ckDb ck.Provider, meter metrics.Meter, mqFactory mq.IFactory, configFactory conf.IConfigLoaderFactory, idgen2 idgen.IIDGenerator, benefit2 benefit.IBenefitService, fileClient fileservice.Client, authCli authservice.Client, userClient userservice.Client, evalClient evaluatorservice.Client, evalSetClient evaluationsetservice.Client, tagClient tagservice.Client, limiterFactory limiter.IRateLimiterFactory, datasetClient datasetservice.Client, redis2 redis.Cmdable, persistentCmdable redis.PersistentCmdable, experimentClient experimentservice.Client, taskProcessor processor.TaskProcessor, aid int32) (*ObservabilityHandler, error) { +func InitObservabilityHandler(ctx context.Context, db2 db.Provider, ckDb ck.Provider, meter metrics.Meter, mqFactory mq.IFactory, configFactory conf.IConfigLoaderFactory, idgen2 idgen.IIDGenerator, benefit2 benefit.IBenefitService, fileClient fileservice.Client, authCli authservice.Client, userClient userservice.Client, evalClient evaluatorservice.Client, evalSetClient evaluationsetservice.Client, tagClient tagservice.Client, limiterFactory limiter.IRateLimiterFactory, datasetClient datasetservice.Client, redis2 redis.Cmdable, persistentCmdable redis.PersistentCmdable, storageProvider storage.IStorageProvider, experimentClient experimentservice.Client, taskProcessor processor.TaskProcessor, aid int32) (*ObservabilityHandler, error) { iTraceApplication, err := application6.InitTraceApplication(db2, ckDb, redis2, persistentCmdable, meter, mqFactory, configFactory, idgen2, fileClient, benefit2, authCli, userClient, evalClient, evalSetClient, tagClient, datasetClient) if err != nil { return nil, err } - iTraceIngestionApplication, err := application6.InitTraceIngestionApplication(configFactory, ckDb, mqFactory, persistentCmdable) + iTraceIngestionApplication, err := application6.InitTraceIngestionApplication(configFactory, storageProvider, ckDb, mqFactory, persistentCmdable) if err != nil { return nil, err } @@ -172,7 +172,7 @@ func InitObservabilityHandler(ctx context.Context, db2 db.Provider, ckDb ck.Prov if err != nil { return nil, err } - iMetricApplication, err := application6.InitMetricApplication(ckDb, configFactory, fileClient, benefit2, authCli) + iMetricApplication, err := application6.InitMetricApplication(ckDb, storageProvider, configFactory, fileClient, benefit2, authCli) if err != nil { return nil, err } diff --git a/backend/go.mod b/backend/go.mod index 166fbfa52..fb997a75f 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -1,8 +1,6 @@ module github.com/coze-dev/coze-loop/backend -go 1.24.1 - -toolchain go1.24.6 +go 1.24.6 replace github.com/apache/thrift => github.com/apache/thrift v0.13.0 @@ -17,9 +15,9 @@ require ( github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 github.com/aws/aws-sdk-go v1.55.7 github.com/baidubce/bce-qianfan-sdk/go/qianfan v0.0.15 - github.com/bytedance/gg v1.0.0 + github.com/bytedance/gg v1.1.0 github.com/bytedance/gopkg v0.1.3 - github.com/bytedance/sonic v1.14.1 + github.com/bytedance/sonic v1.14.2 github.com/cenk/backoff v2.2.1+incompatible github.com/cloudwego/eino v0.3.55 github.com/cloudwego/eino-ext/components/model/ark v0.1.8 @@ -32,13 +30,12 @@ require ( github.com/cloudwego/eino-ext/components/model/qianfan v0.0.0-20250520101807-b2008771903a github.com/cloudwego/eino-ext/components/model/qwen v0.0.0-20250520101807-b2008771903a github.com/cloudwego/eino-ext/libs/acl/openai v0.0.0-20250519084852-38fafa73d9ea - github.com/cloudwego/gopkg v0.1.4 + github.com/cloudwego/gopkg v0.1.6 github.com/cloudwego/hertz v0.10.1 - github.com/cloudwego/kitex v0.13.1 + github.com/cloudwego/kitex v0.15.2 github.com/coocood/freecache v1.2.4 github.com/coreos/go-semver v0.3.0 - github.com/coze-dev/cozeloop-go v0.1.14 - github.com/coze-dev/cozeloop-go/spec v0.1.4-0.20250829072213-3812ddbfb735 + github.com/coze-dev/cozeloop-go/spec v0.1.6 github.com/deatil/go-encoding v1.0.3003 github.com/dimchansky/utfbom v1.1.1 github.com/dolthub/go-mysql-server v0.18.0 @@ -77,18 +74,18 @@ require ( github.com/sirupsen/logrus v1.9.3 github.com/spf13/cast v1.7.1 github.com/spf13/viper v1.20.1 - github.com/stretchr/testify v1.10.0 + github.com/stretchr/testify v1.11.1 github.com/valyala/fasttemplate v1.2.2 github.com/vincent-petithory/dataurl v1.0.0 github.com/volcengine/volcengine-go-sdk v1.1.4 github.com/xeipuuv/gojsonschema v1.2.0 - go.opentelemetry.io/proto/otlp v1.7.1 + go.opentelemetry.io/proto/otlp v1.9.0 go.uber.org/mock v0.4.0 - golang.org/x/crypto v0.40.0 + golang.org/x/crypto v0.41.0 golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476 golang.org/x/sync v0.16.0 - golang.org/x/text v0.27.0 - gonum.org/v1/gonum v0.15.0 + golang.org/x/text v0.28.0 + gonum.org/v1/gonum v0.16.0 google.golang.org/api v0.215.0 gorm.io/datatypes v1.2.5 gorm.io/driver/clickhouse v0.6.1 @@ -99,9 +96,13 @@ require ( gorm.io/plugin/soft_delete v1.2.1 ) -require github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1 // indirect +require github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 // indirect -require github.com/brianvoe/gofakeit/v6 v6.28.0 +require ( + github.com/brianvoe/gofakeit/v6 v6.28.0 + github.com/coze-dev/coze-loop/backend/modules/observability/lib v0.0.0-00010101000000-000000000000 + github.com/coze-dev/cozeloop-go v0.1.16 +) require ( cloud.google.com/go v0.116.0 // indirect @@ -132,19 +133,18 @@ require ( github.com/baidubce/bce-sdk-go v0.9.164 // indirect github.com/bluele/gcache v0.0.2 // indirect github.com/bufbuild/protocompile v0.8.0 // indirect - github.com/bytedance/sonic/loader v0.3.0 // indirect + github.com/bytedance/sonic/loader v0.4.0 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cloudwego/base64x v0.1.6 // indirect github.com/cloudwego/configmanager v0.2.3 // indirect - github.com/cloudwego/dynamicgo v0.6.2 // indirect + github.com/cloudwego/dynamicgo v0.7.0 // indirect github.com/cloudwego/fastpb v0.0.5 // indirect - github.com/cloudwego/frugal v0.2.5 // indirect - github.com/cloudwego/iasm v0.2.0 // indirect + github.com/cloudwego/frugal v0.3.0 // indirect github.com/cloudwego/localsession v0.1.2 // indirect - github.com/cloudwego/netpoll v0.7.0 // indirect + github.com/cloudwego/netpoll v0.7.2 // indirect github.com/cloudwego/runtimex v0.1.1 // indirect - github.com/cloudwego/thriftgo v0.4.1 // indirect + github.com/cloudwego/thriftgo v0.4.3 // indirect github.com/cohesion-org/deepseek-go v1.2.8 // indirect github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect @@ -238,23 +238,23 @@ require ( go.opentelemetry.io/auto/sdk v1.1.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 // indirect - go.opentelemetry.io/otel v1.36.0 - go.opentelemetry.io/otel/metric v1.36.0 // indirect - go.opentelemetry.io/otel/trace v1.36.0 // indirect + go.opentelemetry.io/otel v1.38.0 // indirect + go.opentelemetry.io/otel/metric v1.38.0 // indirect + go.opentelemetry.io/otel/trace v1.38.0 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect golang.org/x/arch v0.15.0 // indirect - golang.org/x/mod v0.25.0 // indirect - golang.org/x/net v0.42.0 // indirect + golang.org/x/mod v0.26.0 // indirect + golang.org/x/net v0.43.0 // indirect golang.org/x/oauth2 v0.30.0 // indirect - golang.org/x/sys v0.34.0 // indirect + golang.org/x/sys v0.35.0 // indirect golang.org/x/time v0.8.0 // indirect - golang.org/x/tools v0.34.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20250728155136-f173205681a0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20250728155136-f173205681a0 // indirect - google.golang.org/grpc v1.74.2 - google.golang.org/protobuf v1.36.6 + golang.org/x/tools v0.35.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 // indirect + google.golang.org/grpc v1.75.1 + google.golang.org/protobuf v1.36.10 gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/src-d/go-errors.v1 v1.0.0 // indirect gopkg.in/yaml.v2 v2.4.0 @@ -264,3 +264,5 @@ require ( gorm.io/rawsql v1.0.3-0.20250401110442-7e49778bc820 stathat.com/c/consistent v1.0.0 // indirect ) + +replace github.com/coze-dev/coze-loop/backend/modules/observability/lib => github.com/coze-dev/coze-loop/backend/modules/observability/lib v0.0.0-20251118131611-99b3d466d529 diff --git a/backend/go.sum b/backend/go.sum index 755515be3..8459bd31b 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -162,17 +162,17 @@ github.com/bufbuild/protocompile v0.8.0 h1:9Kp1q6OkS9L4nM3FYbr8vlJnEwtbpDPQlQOVX github.com/bufbuild/protocompile v0.8.0/go.mod h1:+Etjg4guZoAqzVk2czwEQP12yaxLJ8DxuqCJ9qHdH94= github.com/bugsnag/bugsnag-go v1.4.0/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8= github.com/bugsnag/panicwrap v1.2.0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE= -github.com/bytedance/gg v1.0.0 h1:NIV/doHwRI9L2uqXdUI36TiJDpZ2GP8eehT8tKmRcXc= -github.com/bytedance/gg v1.0.0/go.mod h1:MeGhXyy5K20hNAU9GkMM51sXdm/lsqdU0CxwIiGvZpo= +github.com/bytedance/gg v1.1.0 h1:FSKRxOZeN30w7h6snEbHxzgVMUV7+Xu4gc/Lz1cmBFw= +github.com/bytedance/gg v1.1.0/go.mod h1:MeGhXyy5K20hNAU9GkMM51sXdm/lsqdU0CxwIiGvZpo= github.com/bytedance/gopkg v0.1.1/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM= github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M= github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM= github.com/bytedance/mockey v1.2.14 h1:KZaFgPdiUwW+jOWFieo3Lr7INM1P+6adO3hxZhDswY8= github.com/bytedance/mockey v1.2.14/go.mod h1:1BPHF9sol5R1ud/+0VEHGQq/+i2lN+GTsr3O2Q9IENY= -github.com/bytedance/sonic v1.14.1 h1:FBMC0zVz5XUmE4z9wF4Jey0An5FueFvOsTKKKtwIl7w= -github.com/bytedance/sonic v1.14.1/go.mod h1:gi6uhQLMbTdeP0muCnrjHLeCUPyb70ujhnNlhOylAFc= -github.com/bytedance/sonic/loader v0.3.0 h1:dskwH8edlzNMctoruo8FPTJDF3vLtDT0sXZwvZJyqeA= -github.com/bytedance/sonic/loader v0.3.0/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= +github.com/bytedance/sonic v1.14.2 h1:k1twIoe97C1DtYUo+fZQy865IuHia4PR5RPiuGPPIIE= +github.com/bytedance/sonic v1.14.2/go.mod h1:T80iDELeHiHKSc0C9tubFygiuXoGzrkjKzX2quAx980= +github.com/bytedance/sonic/loader v0.4.0 h1:olZ7lEqcxtZygCK9EKYKADnpQoYkRQxaeY2NYzevs+o= +github.com/bytedance/sonic/loader v0.4.0/go.mod h1:AR4NYCk5DdzZizZ5djGqQ92eEhCCcdf5x77udYiSJRo= github.com/casbin/casbin/v2 v2.37.0/go.mod h1:vByNa/Fchek0KZUgG5wEsl7iFsiviAYKRtgrQfcJqHg= github.com/cenk/backoff v2.2.1+incompatible h1:djdFT7f4gF2ttuzRKPbMOWgZajgesItGLwG5FTQKmmE= github.com/cenk/backoff v2.2.1+incompatible/go.mod h1:7FtoeaSnHoZnmZzz47cM35Y9nSW7tNyaidugnHTaFDE= @@ -198,8 +198,8 @@ github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU= github.com/cloudwego/configmanager v0.2.3 h1:P0YTBgqDBnKeI/VARvut/Dc9Rfxt9Bw1Nv7sk0Ru4u8= github.com/cloudwego/configmanager v0.2.3/go.mod h1:4GeSKjH6JLvKx4/Hrbh5dse8fDqj1n/Up8HfU4wHJ+w= -github.com/cloudwego/dynamicgo v0.6.2 h1:jpb0R27Kh1cNUFsQsOCTchyt9oNG0UvwDvTecEnV+xg= -github.com/cloudwego/dynamicgo v0.6.2/go.mod h1:ZfuIc4tsk8gdsmsoL+3M/q3916xTj+KAVJaXQHSaWiE= +github.com/cloudwego/dynamicgo v0.7.0 h1:Z97rJ/FcccgZ16lzQaZU5X1yzg7yUy42uLRXafqpPXE= +github.com/cloudwego/dynamicgo v0.7.0/go.mod h1:f9le2ULWbFFkQ8WoP+7pGl1zEI2xRLZhaaif6ROLwDw= github.com/cloudwego/eino v0.3.55 h1:lMZrGtEh0k3qykQTLNXSXuAa98OtF2tS43GMHyvN7nA= github.com/cloudwego/eino v0.3.55/go.mod h1:wUjz990apdsaOraOXdh6CdhVXq8DJsOvLsVlxNTcNfY= github.com/cloudwego/eino-ext/components/model/ark v0.1.8 h1:QU0M01WNTVf/63cUjD6S/D1lB+ggvcVH4ntZ+XKg5Lo= @@ -224,24 +224,23 @@ github.com/cloudwego/eino-ext/libs/acl/openai v0.0.0-20250519084852-38fafa73d9ea github.com/cloudwego/eino-ext/libs/acl/openai v0.0.0-20250519084852-38fafa73d9ea/go.mod h1:21bzzKhB1SSBr2jUaEBvNs75ZxSWSfIyM3oF2RB1ELs= github.com/cloudwego/fastpb v0.0.5 h1:vYnBPsfbAtU5TVz5+f9UTlmSCixG9F9vRwaqE0mZPZU= github.com/cloudwego/fastpb v0.0.5/go.mod h1:Bho7aAKBUtT9RPD2cNVkTdx4yQumfSv3If7wYnm1izk= -github.com/cloudwego/frugal v0.2.5 h1:zRICkWpBCQ6TY4QmRf+uINzcHbv7ogCHOM8h7ltPRzM= -github.com/cloudwego/frugal v0.2.5/go.mod h1:nC1U47gswLRiaxv6dybrhZvsDGCfQP9RGiiWC73CnoI= -github.com/cloudwego/gopkg v0.1.4 h1:EoQiCG4sTonTPHxOGE0VlQs+sQR+Hsi2uN0qqwu8O50= +github.com/cloudwego/frugal v0.3.0 h1:tgAP0nytiJuyoIM3V3TDOGzjrSNRAIlNG1HHOAzZ3Cs= +github.com/cloudwego/frugal v0.3.0/go.mod h1:pMk46fFyAwUbW7q7lfdK7c6HsD6bWtu6/3Vhz63CgsY= github.com/cloudwego/gopkg v0.1.4/go.mod h1:FQuXsRWRsSqJLsMVd5SYzp8/Z1y5gXKnVvRrWUOsCMI= +github.com/cloudwego/gopkg v0.1.6 h1:EMlOHg975CxKX1/BtIVYKGW8hxNptTkjjJ7bvfXu4L4= +github.com/cloudwego/gopkg v0.1.6/go.mod h1:FQuXsRWRsSqJLsMVd5SYzp8/Z1y5gXKnVvRrWUOsCMI= github.com/cloudwego/hertz v0.10.1 h1:gTM2JIGO7vmRoaDz71GctyoUE19pXGuznFX55HjGs1g= github.com/cloudwego/hertz v0.10.1/go.mod h1:0sofikwk5YcHCerClgCzcaoamY61JiRwR5G0mAUo+Y0= -github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= -github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= -github.com/cloudwego/kitex v0.13.1 h1:oPJS/hy9gvo0rlfQmJAKJj8F4PMLG74IYzpaPlCRgg8= -github.com/cloudwego/kitex v0.13.1/go.mod h1:eHEp//JKqEnQYFPLifEMOikxuLikEnfVXKKniroLTjA= +github.com/cloudwego/kitex v0.15.2 h1:YeVBnQ4KZ1+lS5rjcJfad+6dLV/nKgIcx5T93eKCwrk= +github.com/cloudwego/kitex v0.15.2/go.mod h1:IiThcGN0SokNWdaoUyh8+yB65zn17mOIWIrRjWTqjq4= github.com/cloudwego/localsession v0.1.2 h1:RBmeLDO5sKr4ujd8iBp5LTMmuVKLdu88jjIneq/fEZ8= github.com/cloudwego/localsession v0.1.2/go.mod h1:J4uams2YT/2d4t7OI6A7NF7EcG8OlHJsOX2LdPbqoyc= -github.com/cloudwego/netpoll v0.7.0 h1:bDrxQaNfijRI1zyGgXHQoE/nYegL0nr+ijO1Norelc4= -github.com/cloudwego/netpoll v0.7.0/go.mod h1:PI+YrmyS7cIr0+SD4seJz3Eo3ckkXdu2ZVKBLhURLNU= +github.com/cloudwego/netpoll v0.7.2 h1:4qDBGQ6CG2SvEXhZSDxMdtqt/NLDxjAVk0PC/biKiJo= +github.com/cloudwego/netpoll v0.7.2/go.mod h1:PI+YrmyS7cIr0+SD4seJz3Eo3ckkXdu2ZVKBLhURLNU= github.com/cloudwego/runtimex v0.1.1 h1:lheZjFOyKpsq8TsGGfmX9/4O7F0TKpWmB8on83k7GE8= github.com/cloudwego/runtimex v0.1.1/go.mod h1:23vL/HGV0W8nSCHbe084AgEBdDV4rvXenEUMnUNvUd8= -github.com/cloudwego/thriftgo v0.4.1 h1:p7wr+YOLlw14Qm8KlJHvEiyo6+LvVjipCyNbg0AwfYg= -github.com/cloudwego/thriftgo v0.4.1/go.mod h1:AdLEJJVGW/ZJYvkkYAZf5SaJH+pA3OyC801WSwqcBwI= +github.com/cloudwego/thriftgo v0.4.3 h1:Ig80u/nQdOiB4K36BG4oqud2f8LMykZkbnk4R4QywiM= +github.com/cloudwego/thriftgo v0.4.3/go.mod h1:/D4zRAEj1t3/Tq1bVGDMnRt3wxpHfalXfZWvq/n4YmY= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= @@ -252,12 +251,12 @@ github.com/coocood/freecache v1.2.4/go.mod h1:RBUWa/Cy+OHdfTGFEhEuE1pMCMX51Ncizj github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/coze-dev/cozeloop-go v0.1.10-0.20250901062520-61d3699b1e83 h1:7Jh4flr9XqvissJtafWhTcs1vcErUcsjNkkniH/szxY= -github.com/coze-dev/cozeloop-go v0.1.10-0.20250901062520-61d3699b1e83/go.mod h1:RMH0F6ZMwZm4ZL92IHLjTf4lmr8QHxYJVPCdz60ZbbI= -github.com/coze-dev/cozeloop-go v0.1.14 h1:Pu6P+G72czlGn9e86aSXpXuRqJvu388fWXN8J/heVOc= -github.com/coze-dev/cozeloop-go v0.1.14/go.mod h1:lM7cmUEZlnAlQYdwfk4Li0SC3RdZ++QMHX75nvKceSc= -github.com/coze-dev/cozeloop-go/spec v0.1.4-0.20250829072213-3812ddbfb735 h1:qxAwjHy0SLQazDO3oGJ8D24vOeM2Oz2+n27bNPegBls= -github.com/coze-dev/cozeloop-go/spec v0.1.4-0.20250829072213-3812ddbfb735/go.mod h1:/f3BrWehffwXIpd4b5rYIqktLd/v5dlLBw0h9F/LQIU= +github.com/coze-dev/coze-loop/backend/modules/observability/lib v0.0.0-20251118131611-99b3d466d529 h1:jFs6HE0WNlOis6wOn5Lht0Jp0XX/5rjNNtSEjbjHCYc= +github.com/coze-dev/coze-loop/backend/modules/observability/lib v0.0.0-20251118131611-99b3d466d529/go.mod h1:shS2ZraVjQVO7xf/fNmEFXaf40c/V9aLo6YvvCHKdRw= +github.com/coze-dev/cozeloop-go v0.1.16 h1:SUuhrP5TH+t+loRHbKnZtIRRhDQ4UB4P6xJSdQHmAHU= +github.com/coze-dev/cozeloop-go v0.1.16/go.mod h1:lM7cmUEZlnAlQYdwfk4Li0SC3RdZ++QMHX75nvKceSc= +github.com/coze-dev/cozeloop-go/spec v0.1.6 h1:lStq3CfvTwBn0y281X4KnhW7mtf9g/XIcccGEDSWTD0= +github.com/coze-dev/cozeloop-go/spec v0.1.6/go.mod h1:/f3BrWehffwXIpd4b5rYIqktLd/v5dlLBw0h9F/LQIU= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 h1:iwZdTE0PVqJCos1vaoKsclOGD3ADKpshg3SRtYBbwso= @@ -276,7 +275,6 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/r github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= -github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= github.com/dolthub/flatbuffers/v23 v23.3.3-dh.2 h1:u3PMzfF8RkKd3lB9pZ2bfn0qEG+1Gms9599cr0REMww= github.com/dolthub/flatbuffers/v23 v23.3.3-dh.2/go.mod h1:mIEZOHnFx4ZMQeawhw9rhsj+0zwQj7adVsnBX7t+eKY= @@ -475,7 +473,6 @@ github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5/go.mod h1:vavhavw2zAx github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/s2a-go v0.1.8 h1:zZDs9gcbt9ZPLV0ndSyQk6Kacx2g/X+SKYovpnz3SMM= github.com/google/s2a-go v0.1.8/go.mod h1:6iNWHTpQ+nfNRN5E00MSdfDwVesa8hhS32PhPO8deJA= -github.com/google/subcommands v1.2.0 h1:vWQspBTo2nEqTUFita5/KeEWlUL8kQObDFbub/EN9oE= github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -501,8 +498,8 @@ github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+ github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1 h1:X5VWvz21y3gzm9Nw/kaUeku/1+uBhcekkmy4IkffJww= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1/go.mod h1:Zanoh4+gvIgluNqcfMVTJueD4wSS5hT7zTt4Mrutd90= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 h1:8Tjv8EJ+pM1xP8mK6egEbD1OgnVTyacbefKhmbLhIhU= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2/go.mod h1:pkJQ2tZHJ0aFOVEEot6oZmaVEZcRme73eIFmhiVuRWs= github.com/hashicorp/consul/api v1.10.1/go.mod h1:XjsvQN+RJGWI2TWy1/kqaE16HrR2J/FWgkYjdZQsX9M= github.com/hashicorp/consul/sdk v0.8.0/go.mod h1:GBvyrGALthsZObzUGsfgHZQDXjg4lOjagTIwIR1vPms= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -611,10 +608,8 @@ github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47e github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= -github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= -github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= @@ -891,8 +886,9 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/tetratelabs/wazero v1.1.0 h1:EByoAhC+QcYpwSZJSs/aV0uokxPwBgKxfiokSUwAknQ= @@ -973,19 +969,19 @@ go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.5 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0/go.mod h1:B9yO6b04uB80CzjedvewuqDhxJxi11s7/GtiGa8bAjI= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 h1:TT4fX+nBOA/+LUkobKGW1ydGcn+G3vRw9+g5HwCphpk= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0/go.mod h1:L7UH0GbB0p47T4Rri3uHjbpCFYrVrwc1I25QhNPiGK8= -go.opentelemetry.io/otel v1.36.0 h1:UumtzIklRBY6cI/lllNZlALOF5nNIzJVb16APdvgTXg= -go.opentelemetry.io/otel v1.36.0/go.mod h1:/TcFMXYjyRNh8khOAO9ybYkqaDBb/70aVwkNML4pP8E= -go.opentelemetry.io/otel/metric v1.36.0 h1:MoWPKVhQvJ+eeXWHFBOPoBOi20jh6Iq2CcCREuTYufE= -go.opentelemetry.io/otel/metric v1.36.0/go.mod h1:zC7Ks+yeyJt4xig9DEw9kuUFe5C3zLbVjV2PzT6qzbs= -go.opentelemetry.io/otel/sdk v1.36.0 h1:b6SYIuLRs88ztox4EyrvRti80uXIFy+Sqzoh9kFULbs= -go.opentelemetry.io/otel/sdk v1.36.0/go.mod h1:+lC+mTgD+MUWfjJubi2vvXWcVxyr9rmlshZni72pXeY= -go.opentelemetry.io/otel/sdk/metric v1.36.0 h1:r0ntwwGosWGaa0CrSt8cuNuTcccMXERFwHX4dThiPis= -go.opentelemetry.io/otel/sdk/metric v1.36.0/go.mod h1:qTNOhFDfKRwX0yXOqJYegL5WRaW376QbB7P4Pb0qva4= -go.opentelemetry.io/otel/trace v1.36.0 h1:ahxWNuqZjpdiFAyrIoQ4GIiAIhxAunQR6MUoKrsNd4w= -go.opentelemetry.io/otel/trace v1.36.0/go.mod h1:gQ+OnDZzrybY4k4seLzPAWNwVBBVlF2szhehOBB/tGA= +go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8= +go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM= +go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA= +go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI= +go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI= +go.opentelemetry.io/otel/sdk v1.37.0/go.mod h1:VredYzxUvuo2q3WRcDnKDjbdvmO0sCzOvVAiY+yUkAg= +go.opentelemetry.io/otel/sdk/metric v1.37.0 h1:90lI228XrB9jCMuSdA0673aubgRobVZFhbjxHHspCPc= +go.opentelemetry.io/otel/sdk/metric v1.37.0/go.mod h1:cNen4ZWfiD37l5NhS+Keb5RXVWZWpRE+9WyVCpbo5ps= +go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE= +go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= -go.opentelemetry.io/proto/otlp v1.7.1 h1:gTOMpGDb0WTBOP8JaO72iL3auEZhVmAQg4ipjOVAtj4= -go.opentelemetry.io/proto/otlp v1.7.1/go.mod h1:b2rVh6rfI/s2pHWNlB7ILJcRALpcNDzKhACevjI+ZnE= +go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A= +go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4= go.uber.org/atomic v1.5.1/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= @@ -1032,8 +1028,8 @@ golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliY golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= -golang.org/x/crypto v0.40.0 h1:r4x+VvoG5Fm+eJcxMaY8CQM7Lb0l1lsmjGBQ6s8BfKM= -golang.org/x/crypto v0.40.0/go.mod h1:Qr1vMER5WyS2dfPHAlsOj01wgLbsyWtFn/aY+5+ZdxY= +golang.org/x/crypto v0.41.0 h1:WKYxWedPGCTVVl5+WHSSrOBT0O8lx32+zxmHxijgXp4= +golang.org/x/crypto v0.41.0/go.mod h1:pO5AFd7FA68rFak7rOAGVuygIISepHftHnr8dr6+sUc= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1076,8 +1072,8 @@ golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/mod v0.25.0 h1:n7a+ZbQKQA/Ysbyb0/6IbB1H/X41mKgbhfv7AfG/44w= -golang.org/x/mod v0.25.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= +golang.org/x/mod v0.26.0 h1:EGMPT//Ezu+ylkCijjPc+f4Aih7sZvaAr+O3EHBxvZg= +golang.org/x/mod v0.26.0/go.mod h1:/j6NAhSk8iQ723BGAUyoAcn7SlD7s15Dp9Nd/SfeaFQ= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1128,8 +1124,8 @@ golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= -golang.org/x/net v0.42.0 h1:jzkYrhi3YQWD6MLBJcsklgQsoAcw89EcZbJw8Z614hs= -golang.org/x/net v0.42.0/go.mod h1:FF1RA5d3u7nAYA4z2TkclSCKh68eSXtiFwcWQpPXdt8= +golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= +golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1225,8 +1221,8 @@ golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= -golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= +golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= @@ -1236,8 +1232,8 @@ golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk= -golang.org/x/term v0.33.0 h1:NuFncQrRcaRvVmgRkvM3j/F00gWIAlcmlB8ACEKmGIg= -golang.org/x/term v0.33.0/go.mod h1:s18+ql9tYWp1IfpV9DmCtQDDSRBUjKaw9M1eAv5UeF0= +golang.org/x/term v0.34.0 h1:O/2T7POpk0ZZ7MAzMeWFSg6S5IpWd/RXDlM9hgM3DR4= +golang.org/x/term v0.34.0/go.mod h1:5jC53AEywhIVebHgPVeg0mj8OD3VO9OzclacVrqpaAw= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1251,8 +1247,8 @@ golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4= -golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU= +golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= +golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1316,16 +1312,16 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps= -golang.org/x/tools v0.34.0 h1:qIpSLOxeCYGg9TrcJokLBG4KFA6d795g0xkBkiESGlo= -golang.org/x/tools v0.34.0/go.mod h1:pAP9OwEaY1CAW3HOmg3hLZC5Z0CCmzjAF2UQMSqNARg= +golang.org/x/tools v0.35.0 h1:mBffYraMEf7aa0sB+NuKnuCy8qI/9Bughn8dC2Gu5r0= +golang.org/x/tools v0.35.0/go.mod h1:NKdj5HkL/73byiZSJjqJgKn3ep7KjFkBOkR/Hps3VPw= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= -gonum.org/v1/gonum v0.15.0 h1:2lYxjRbTYyxkJxlhC+LvJIx3SsANPdRybu1tGj9/OrQ= -gonum.org/v1/gonum v0.15.0/go.mod h1:xzZVBJBtS+Mz4q0Yl2LJTk+OxOg4jiXZ7qBoM0uISGo= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= @@ -1384,10 +1380,10 @@ google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= google.golang.org/genproto v0.0.0-20210917145530-b395a37504d4/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto/googleapis/api v0.0.0-20250728155136-f173205681a0 h1:0UOBWO4dC+e51ui0NFKSPbkHHiQ4TmrEfEZMLDyRmY8= -google.golang.org/genproto/googleapis/api v0.0.0-20250728155136-f173205681a0/go.mod h1:8ytArBbtOy2xfht+y2fqKd5DRDJRUQhqbyEnQ4bDChs= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250728155136-f173205681a0 h1:MAKi5q709QWfnkkpNQ0M12hYJ1+e8qYVDyowc4U1XZM= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250728155136-f173205681a0/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= +google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 h1:BIRfGDEjiHRrk0QKZe3Xv2ieMhtgRGeLcZQ0mIVn4EY= +google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5/go.mod h1:j3QtIyytwqGr1JUDtYXwtMXWPKsEa5LtzIFN1Wn5WvE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 h1:eaY8u2EuxbRv7c3NiGK0/NedzVsCcV6hDuU5qPX5EGE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5/go.mod h1:M4/wBTSeyLxupu3W3tJtOgB14jILAS/XWPSSa3TAlJc= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1405,8 +1401,8 @@ google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.74.2 h1:WoosgB65DlWVC9FqI82dGsZhWFNBSLjQ84bjROOpMu4= -google.golang.org/grpc v1.74.2/go.mod h1:CtQ+BGjaAIXHs/5YS3i473GqwBBa1zGQNevxdeBEXrM= +google.golang.org/grpc v1.75.1 h1:/ODCNEuf9VghjgO3rqLcfg8fiOP0nSluljWFlDxELLI= +google.golang.org/grpc v1.75.1/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -1421,8 +1417,8 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= -google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= +google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= +google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/cenkalti/backoff.v1 v1.1.0 h1:Arh75ttbsvlpVA7WtVpH4u9h6Zl46xuptxqLxPiSo4Y= gopkg.in/cenkalti/backoff.v1 v1.1.0/go.mod h1:J6Vskwqd+OMVJl8C33mmtxTBs2gyzfv7UDAkHu8BrjI= @@ -1494,7 +1490,6 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= diff --git a/backend/kitex_gen/coze/loop/observability/domain/common/common.go b/backend/kitex_gen/coze/loop/observability/domain/common/common.go index 51650efc3..ce2fce1a4 100644 --- a/backend/kitex_gen/coze/loop/observability/domain/common/common.go +++ b/backend/kitex_gen/coze/loop/observability/domain/common/common.go @@ -27,6 +27,8 @@ const ( PlatformTypeVeADK = "veadk" + PlatformTypeVeAgentkit = "ve_agentkit" + PlatformTypeLoopAll = "loop_all" PlatformTypeInnerCozeloop = "inner_cozeloop" diff --git a/backend/modules/observability/application/convertor/page_test.go b/backend/modules/observability/application/convertor/page_test.go new file mode 100644 index 000000000..661a0df8b --- /dev/null +++ b/backend/modules/observability/application/convertor/page_test.go @@ -0,0 +1,122 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 + +package convertor + +import ( + "testing" + + "github.com/coze-dev/coze-loop/backend/kitex_gen/coze/loop/observability/domain/common" + entity "github.com/coze-dev/coze-loop/backend/modules/observability/domain/trace/entity/common" + "github.com/coze-dev/coze-loop/backend/pkg/lang/ptr" + "github.com/stretchr/testify/assert" +) + +func TestOrderByDTO2DO(t *testing.T) { + t.Parallel() + + t.Run("nil input", func(t *testing.T) { + t.Parallel() + result := OrderByDTO2DO(nil) + assert.Nil(t, result) + }) + + t.Run("valid input", func(t *testing.T) { + t.Parallel() + orderBy := &common.OrderBy{ + Field: ptr.Of("created_at"), + IsAsc: ptr.Of(true), + } + result := OrderByDTO2DO(orderBy) + assert.NotNil(t, result) + assert.Equal(t, "created_at", result.Field) + assert.True(t, result.IsAsc) + }) + + t.Run("descending order", func(t *testing.T) { + t.Parallel() + orderBy := &common.OrderBy{ + Field: ptr.Of("updated_at"), + IsAsc: ptr.Of(false), + } + result := OrderByDTO2DO(orderBy) + assert.NotNil(t, result) + assert.Equal(t, "updated_at", result.Field) + assert.False(t, result.IsAsc) + }) +} + +func TestOrderByDO2DTO(t *testing.T) { + t.Parallel() + + t.Run("nil input", func(t *testing.T) { + t.Parallel() + result := OrderByDO2DTO(nil) + assert.Nil(t, result) + }) + + t.Run("valid input", func(t *testing.T) { + t.Parallel() + orderBy := &entity.OrderBy{ + Field: "name", + IsAsc: true, + } + result := OrderByDO2DTO(orderBy) + assert.NotNil(t, result) + assert.NotNil(t, result.Field) + assert.Equal(t, "name", *result.Field) + assert.NotNil(t, result.IsAsc) + assert.True(t, *result.IsAsc) + }) + + t.Run("descending order", func(t *testing.T) { + t.Parallel() + orderBy := &entity.OrderBy{ + Field: "id", + IsAsc: false, + } + result := OrderByDO2DTO(orderBy) + assert.NotNil(t, result) + assert.NotNil(t, result.Field) + assert.Equal(t, "id", *result.Field) + assert.NotNil(t, result.IsAsc) + assert.False(t, *result.IsAsc) + }) +} + +func TestOrderByConversionRoundTrip(t *testing.T) { + t.Parallel() + + t.Run("round trip conversion", func(t *testing.T) { + t.Parallel() + original := &common.OrderBy{ + Field: ptr.Of("test_field"), + IsAsc: ptr.Of(true), + } + // DTO -> DO -> DTO + do := OrderByDTO2DO(original) + assert.NotNil(t, do) + + result := OrderByDO2DTO(do) + assert.NotNil(t, result) + assert.Equal(t, *original.Field, *result.Field) + assert.Equal(t, *original.IsAsc, *result.IsAsc) + }) + + t.Run("round trip with entity", func(t *testing.T) { + t.Parallel() + original := &entity.OrderBy{ + Field: "entity_field", + IsAsc: false, + } + + // DO -> DTO -> DO + dto := OrderByDO2DTO(original) + assert.NotNil(t, dto) + + result := OrderByDTO2DO(dto) + assert.NotNil(t, result) + assert.Equal(t, original.Field, result.Field) + assert.Equal(t, original.IsAsc, result.IsAsc) + }) +} diff --git a/backend/modules/observability/application/convertor/task/filter_test.go b/backend/modules/observability/application/convertor/task/filter_test.go new file mode 100644 index 000000000..2ff742b6c --- /dev/null +++ b/backend/modules/observability/application/convertor/task/filter_test.go @@ -0,0 +1,272 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 + +package task + +import ( + "testing" + + "github.com/coze-dev/coze-loop/backend/kitex_gen/coze/loop/observability/domain/filter" + "github.com/coze-dev/coze-loop/backend/modules/observability/domain/task/entity" + "github.com/coze-dev/coze-loop/backend/pkg/lang/ptr" + "github.com/stretchr/testify/assert" +) + +func TestTaskFiltersDTO2DO(t *testing.T) { + t.Parallel() + + t.Run("nil input", func(t *testing.T) { + t.Parallel() + result := TaskFiltersDTO2DO(nil) + assert.Nil(t, result) + }) + + t.Run("empty filters", func(t *testing.T) { + t.Parallel() + filters := &filter.TaskFilterFields{ + FilterFields: []*filter.TaskFilterField{}, + } + result := TaskFiltersDTO2DO(filters) + assert.NotNil(t, result) + assert.Empty(t, result.FilterFields) + }) + + t.Run("with query relation", func(t *testing.T) { + t.Parallel() + relation := filter.QueryRelationAnd + filters := &filter.TaskFilterFields{ + QueryAndOr: &relation, + FilterFields: []*filter.TaskFilterField{ + { + FieldName: ptr.Of("name"), + Values: []string{"test"}, + }, + }, + } + result := TaskFiltersDTO2DO(filters) + assert.NotNil(t, result) + assert.NotNil(t, result.QueryAndOr) + assert.Equal(t, entity.QueryRelationAnd, *result.QueryAndOr) + assert.Len(t, result.FilterFields, 1) + }) + + t.Run("with nil filter field", func(t *testing.T) { + t.Parallel() + filters := &filter.TaskFilterFields{ + FilterFields: []*filter.TaskFilterField{ + { + FieldName: ptr.Of("field1"), + Values: []string{"value1"}, + }, + nil, + { + FieldName: ptr.Of("field2"), + Values: []string{"value2"}, + }, + }, + } + result := TaskFiltersDTO2DO(filters) + assert.NotNil(t, result) + assert.Len(t, result.FilterFields, 2) // nil field should be skipped + }) + + t.Run("complete filter field", func(t *testing.T) { + t.Parallel() + fieldType := filter.FieldTypeString + queryType := filter.QueryTypeEq + relation := filter.QueryRelationOr + filters := &filter.TaskFilterFields{ + QueryAndOr: &relation, + FilterFields: []*filter.TaskFilterField{ + { + FieldName: ptr.Of("task_name"), + FieldType: &fieldType, + QueryType: &queryType, + QueryAndOr: &relation, + Values: []string{"test_task", "another_task"}, + SubFilter: &filter.TaskFilterField{ + FieldName: ptr.Of("sub_field"), + Values: []string{"sub_value"}, + }, + }, + }, + } + result := TaskFiltersDTO2DO(filters) + assert.NotNil(t, result) + assert.Equal(t, entity.QueryRelationOr, *result.QueryAndOr) + assert.Len(t, result.FilterFields, 1) + + field := result.FilterFields[0] + assert.Equal(t, entity.TaskFieldName("task_name"), *field.FieldName) + assert.Equal(t, entity.FieldTypeString, *field.FieldType) + assert.Equal(t, entity.QueryTypeEq, *field.QueryType) + assert.Equal(t, entity.QueryRelationOr, *field.QueryAndOr) + assert.Equal(t, []string{"test_task", "another_task"}, field.Values) + assert.NotNil(t, field.SubFilter) + assert.Equal(t, entity.TaskFieldName("sub_field"), *field.SubFilter.FieldName) + }) +} + +func TestTaskFiltersDO2DTO(t *testing.T) { + t.Parallel() + + t.Run("nil input", func(t *testing.T) { + t.Parallel() + result := TaskFiltersDO2DTO(nil) + assert.Nil(t, result) + }) + + t.Run("empty filters", func(t *testing.T) { + t.Parallel() + filters := &entity.TaskFilterFields{ + FilterFields: []*entity.TaskFilterField{}, + } + result := TaskFiltersDO2DTO(filters) + assert.NotNil(t, result) + assert.Empty(t, result.FilterFields) + }) + + t.Run("with query relation", func(t *testing.T) { + t.Parallel() + relation := entity.QueryRelationOr + filters := &entity.TaskFilterFields{ + QueryAndOr: &relation, + FilterFields: []*entity.TaskFilterField{ + { + FieldName: func() *entity.TaskFieldName { n := entity.TaskFieldName("status"); return &n }(), + Values: []string{"running"}, + }, + }, + } + result := TaskFiltersDO2DTO(filters) + assert.NotNil(t, result) + assert.NotNil(t, result.QueryAndOr) + assert.Equal(t, filter.QueryRelationOr, *result.QueryAndOr) + assert.Len(t, result.FilterFields, 1) + }) + + t.Run("with nil filter field", func(t *testing.T) { + t.Parallel() + filters := &entity.TaskFilterFields{ + FilterFields: []*entity.TaskFilterField{ + { + FieldName: func() *entity.TaskFieldName { n := entity.TaskFieldName("field1"); return &n }(), + Values: []string{"value1"}, + }, + nil, + { + FieldName: func() *entity.TaskFieldName { n := entity.TaskFieldName("field2"); return &n }(), + Values: []string{"value2"}, + }, + }, + } + result := TaskFiltersDO2DTO(filters) + assert.NotNil(t, result) + assert.Len(t, result.FilterFields, 2) // nil field should be skipped + }) + + t.Run("complete filter field", func(t *testing.T) { + t.Parallel() + fieldType := entity.FieldTypeLong + queryType := entity.QueryTypeGte + relation := entity.QueryRelationAnd + + filters := &entity.TaskFilterFields{ + QueryAndOr: &relation, + FilterFields: []*entity.TaskFilterField{ + { + FieldName: func() *entity.TaskFieldName { n := entity.TaskFieldName("task_id"); return &n }(), + FieldType: &fieldType, + QueryType: &queryType, + QueryAndOr: &relation, + Values: []string{"123", "456"}, + SubFilter: &entity.TaskFilterField{ + FieldName: func() *entity.TaskFieldName { n := entity.TaskFieldName("sub_field"); return &n }(), + Values: []string{"sub_value"}, + }, + }, + }, + } + result := TaskFiltersDO2DTO(filters) + assert.NotNil(t, result) + assert.Equal(t, filter.QueryRelationAnd, *result.QueryAndOr) + assert.Len(t, result.FilterFields, 1) + + field := result.FilterFields[0] + assert.Equal(t, "task_id", *field.FieldName) + assert.Equal(t, filter.FieldTypeLong, *field.FieldType) + assert.Equal(t, filter.QueryTypeGte, *field.QueryType) + assert.Equal(t, filter.QueryRelationAnd, *field.QueryAndOr) + assert.Equal(t, []string{"123", "456"}, field.Values) + assert.NotNil(t, field.SubFilter) + assert.Equal(t, "sub_field", *field.SubFilter.FieldName) + }) +} + +func TestTaskFiltersConversionRoundTrip(t *testing.T) { + t.Parallel() + + t.Run("round trip DTO to DO to DTO", func(t *testing.T) { + t.Parallel() + relation := filter.QueryRelationAnd + fieldType := filter.FieldTypeString + queryType := filter.QueryTypeIn + + original := &filter.TaskFilterFields{ + QueryAndOr: &relation, + FilterFields: []*filter.TaskFilterField{ + { + FieldName: ptr.Of("name"), + FieldType: &fieldType, + QueryType: &queryType, + Values: []string{"test", "prod"}, + }, + }, + } + + // DTO -> DO -> DTO + do := TaskFiltersDTO2DO(original) + assert.NotNil(t, do) + + result := TaskFiltersDO2DTO(do) + assert.NotNil(t, result) + assert.Equal(t, *original.QueryAndOr, *result.QueryAndOr) + assert.Len(t, result.FilterFields, 1) + assert.Equal(t, *original.FilterFields[0].FieldName, *result.FilterFields[0].FieldName) + assert.Equal(t, *original.FilterFields[0].FieldType, *result.FilterFields[0].FieldType) + assert.Equal(t, *original.FilterFields[0].QueryType, *result.FilterFields[0].QueryType) + assert.Equal(t, original.FilterFields[0].Values, result.FilterFields[0].Values) + }) + + t.Run("round trip DO to DTO to DO", func(t *testing.T) { + t.Parallel() + relation := entity.QueryRelationOr + fieldType := entity.FieldTypeDouble + queryType := entity.QueryTypeLt + + original := &entity.TaskFilterFields{ + QueryAndOr: &relation, + FilterFields: []*entity.TaskFilterField{ + { + FieldName: func() *entity.TaskFieldName { n := entity.TaskFieldName("duration"); return &n }(), + FieldType: &fieldType, + QueryType: &queryType, + Values: []string{"100.5", "200.7"}, + }, + }, + } + + // DO -> DTO -> DO + dto := TaskFiltersDO2DTO(original) + assert.NotNil(t, dto) + + result := TaskFiltersDTO2DO(dto) + assert.NotNil(t, result) + assert.Equal(t, *original.QueryAndOr, *result.QueryAndOr) + assert.Len(t, result.FilterFields, 1) + assert.Equal(t, *original.FilterFields[0].FieldName, *result.FilterFields[0].FieldName) + assert.Equal(t, *original.FilterFields[0].FieldType, *result.FilterFields[0].FieldType) + assert.Equal(t, *original.FilterFields[0].QueryType, *result.FilterFields[0].QueryType) + assert.Equal(t, original.FilterFields[0].Values, result.FilterFields[0].Values) + }) +} diff --git a/backend/modules/observability/application/convertor/trace/span.go b/backend/modules/observability/application/convertor/trace/span.go index 246f03d4c..c15b87eb0 100644 --- a/backend/modules/observability/application/convertor/trace/span.go +++ b/backend/modules/observability/application/convertor/trace/span.go @@ -11,6 +11,7 @@ import ( "github.com/coze-dev/coze-loop/backend/modules/observability/domain/component/rpc" "github.com/coze-dev/coze-loop/backend/modules/observability/domain/trace/entity/common" "github.com/coze-dev/coze-loop/backend/modules/observability/domain/trace/entity/loop_span" + "github.com/coze-dev/coze-loop/backend/modules/observability/lib/otel" "github.com/coze-dev/coze-loop/backend/pkg/lang/ptr" "github.com/coze-dev/coze-loop/backend/pkg/lang/slices" time_util "github.com/coze-dev/coze-loop/backend/pkg/time" @@ -243,3 +244,40 @@ func fieldTypeDTO2DO(fieldType *filter.FieldType) loop_span.FieldType { } return loop_span.FieldType(*fieldType) } + +func OtelSpans2LoopSpans(spans []*otel.LoopSpan) []*loop_span.Span { + result := make([]*loop_span.Span, 0) + for i := range spans { + result = append(result, OtelSpan2LoopSpan(spans[i])) + } + return result +} + +func OtelSpan2LoopSpan(span *otel.LoopSpan) *loop_span.Span { + return &loop_span.Span{ + StartTime: span.StartTime, + SpanID: span.SpanID, + ParentID: span.ParentID, + TraceID: span.TraceID, + DurationMicros: span.DurationMicros, + CallType: span.CallType, + PSM: span.PSM, + LogID: span.LogID, + WorkspaceID: span.WorkspaceID, + SpanName: span.SpanName, + SpanType: span.SpanType, + Method: span.Method, + StatusCode: span.StatusCode, + Input: span.Input, + Output: span.Output, + ObjectStorage: span.ObjectStorage, + SystemTagsString: span.SystemTagsString, + SystemTagsLong: span.SystemTagsLong, + SystemTagsDouble: span.SystemTagsDouble, + TagsString: span.TagsString, + TagsLong: span.TagsLong, + TagsDouble: span.TagsDouble, + TagsBool: span.TagsBool, + TagsByte: span.TagsByte, + } +} diff --git a/backend/modules/observability/application/convertor/trace/span_test.go b/backend/modules/observability/application/convertor/trace/span_test.go index db5c422cb..1cb1c773a 100755 --- a/backend/modules/observability/application/convertor/trace/span_test.go +++ b/backend/modules/observability/application/convertor/trace/span_test.go @@ -9,6 +9,7 @@ import ( "github.com/coze-dev/coze-loop/backend/kitex_gen/coze/loop/observability/domain/filter" kitexspan "github.com/coze-dev/coze-loop/backend/kitex_gen/coze/loop/observability/domain/span" "github.com/coze-dev/coze-loop/backend/modules/observability/domain/trace/entity/loop_span" + "github.com/coze-dev/coze-loop/backend/modules/observability/lib/otel" "github.com/coze-dev/coze-loop/backend/pkg/lang/ptr" "github.com/stretchr/testify/assert" ) @@ -337,3 +338,161 @@ func TestFieldTypeDTO2DO(t *testing.T) { assert.Equal(t, loop_span.FieldType(fieldType), fieldTypeDTO2DO(&fieldType)) assert.Equal(t, loop_span.FieldTypeString, fieldTypeDTO2DO(nil)) } + +func TestOtelSpan2LoopSpan(t *testing.T) { + tests := []struct { + name string + span *otel.LoopSpan + }{ + { + name: "complete otel span conversion", + span: &otel.LoopSpan{ + StartTime: 1234567890, + SpanID: "span-123", + ParentID: "parent-456", + TraceID: "trace-789", + DurationMicros: 987654321, + CallType: "http", + PSM: "test-service", + LogID: "log-123", + WorkspaceID: "workspace-456", + SpanName: "test-span", + SpanType: "model", + Method: "POST", + StatusCode: 0, + Input: `{"input": "test"}`, + Output: `{"output": "result"}`, + ObjectStorage: "tos://bucket/key", + SystemTagsString: map[string]string{ + "sys_str": "value1", + }, + SystemTagsLong: map[string]int64{ + "sys_long": 123, + }, + SystemTagsDouble: map[string]float64{ + "sys_double": 1.23, + }, + TagsString: map[string]string{ + "tag_str": "value2", + }, + TagsLong: map[string]int64{ + "tag_long": 456, + }, + TagsDouble: map[string]float64{ + "tag_double": 4.56, + }, + TagsBool: map[string]bool{ + "tag_bool": true, + }, + TagsByte: map[string]string{ + "tag_bytes": "010101", + }, + }, + }, + { + name: "minimal otel span conversion", + span: &otel.LoopSpan{ + StartTime: 0, + SpanID: "minimal", + ParentID: "", + TraceID: "trace-min", + WorkspaceID: "ws", + SpanName: "min-span", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := OtelSpan2LoopSpan(tt.span) + + assert.NotNil(t, result) + assert.Equal(t, tt.span.StartTime, result.StartTime) + assert.Equal(t, tt.span.SpanID, result.SpanID) + assert.Equal(t, tt.span.ParentID, result.ParentID) + assert.Equal(t, tt.span.TraceID, result.TraceID) + assert.Equal(t, tt.span.DurationMicros, result.DurationMicros) + assert.Equal(t, tt.span.CallType, result.CallType) + assert.Equal(t, tt.span.PSM, result.PSM) + assert.Equal(t, tt.span.LogID, result.LogID) + assert.Equal(t, tt.span.WorkspaceID, result.WorkspaceID) + assert.Equal(t, tt.span.SpanName, result.SpanName) + assert.Equal(t, tt.span.SpanType, result.SpanType) + assert.Equal(t, tt.span.Method, result.Method) + assert.Equal(t, tt.span.StatusCode, result.StatusCode) + assert.Equal(t, tt.span.Input, result.Input) + assert.Equal(t, tt.span.Output, result.Output) + assert.Equal(t, tt.span.ObjectStorage, result.ObjectStorage) + assert.Equal(t, tt.span.SystemTagsString, result.SystemTagsString) + assert.Equal(t, tt.span.SystemTagsLong, result.SystemTagsLong) + assert.Equal(t, tt.span.SystemTagsDouble, result.SystemTagsDouble) + assert.Equal(t, tt.span.TagsString, result.TagsString) + assert.Equal(t, tt.span.TagsLong, result.TagsLong) + assert.Equal(t, tt.span.TagsDouble, result.TagsDouble) + assert.Equal(t, tt.span.TagsBool, result.TagsBool) + assert.Equal(t, tt.span.TagsByte, result.TagsByte) + }) + } +} + +func TestOtelSpans2LoopSpans(t *testing.T) { + tests := []struct { + name string + spans []*otel.LoopSpan + }{ + { + name: "multiple otel spans conversion", + spans: []*otel.LoopSpan{ + { + SpanID: "span-1", + TraceID: "trace-1", + SpanName: "span1", + WorkspaceID: "ws1", + }, + { + SpanID: "span-2", + TraceID: "trace-2", + SpanName: "span2", + WorkspaceID: "ws2", + }, + { + SpanID: "span-3", + TraceID: "trace-3", + SpanName: "span3", + WorkspaceID: "ws3", + }, + }, + }, + { + name: "empty spans list", + spans: []*otel.LoopSpan{}, + }, + { + name: "nil spans list", + spans: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := OtelSpans2LoopSpans(tt.spans) + + if tt.spans == nil { + assert.NotNil(t, result) + assert.Len(t, result, 0) + return + } + + assert.NotNil(t, result) + assert.Len(t, result, len(tt.spans)) + + for i, span := range tt.spans { + assert.NotNil(t, result[i]) + assert.Equal(t, span.SpanID, result[i].SpanID) + assert.Equal(t, span.TraceID, result[i].TraceID) + assert.Equal(t, span.SpanName, result[i].SpanName) + assert.Equal(t, span.WorkspaceID, result[i].WorkspaceID) + } + }) + } +} diff --git a/backend/modules/observability/application/openapi.go b/backend/modules/observability/application/openapi.go index 0c153966b..db8bc19e8 100644 --- a/backend/modules/observability/application/openapi.go +++ b/backend/modules/observability/application/openapi.go @@ -18,6 +18,7 @@ import ( "github.com/bytedance/sonic" "github.com/coze-dev/coze-loop/backend/kitex_gen/base" "github.com/coze-dev/coze-loop/backend/modules/observability/domain/component/collector" + "github.com/coze-dev/coze-loop/backend/modules/observability/lib/otel" coltracepb "go.opentelemetry.io/proto/otlp/collector/trace/v1" "google.golang.org/protobuf/proto" @@ -28,17 +29,15 @@ import ( "github.com/coze-dev/coze-loop/backend/kitex_gen/coze/loop/observability/trace" "github.com/coze-dev/coze-loop/backend/modules/observability/domain/component/metrics" - "github.com/coze-dev/coze-loop/backend/modules/observability/application/utils" - "github.com/coze-dev/coze-loop/backend/modules/observability/domain/component/config" - "github.com/coze-dev/coze-loop/backend/modules/observability/domain/component/tenant" - "github.com/coze-dev/coze-loop/backend/modules/observability/domain/component/workspace" - "github.com/coze-dev/coze-loop/backend/modules/observability/domain/trace/entity/otel" - "github.com/coze-dev/coze-loop/backend/infra/external/benefit" "github.com/coze-dev/coze-loop/backend/infra/middleware/session" "github.com/coze-dev/coze-loop/backend/kitex_gen/coze/loop/observability/openapi" tconv "github.com/coze-dev/coze-loop/backend/modules/observability/application/convertor/trace" + "github.com/coze-dev/coze-loop/backend/modules/observability/application/utils" + "github.com/coze-dev/coze-loop/backend/modules/observability/domain/component/config" "github.com/coze-dev/coze-loop/backend/modules/observability/domain/component/rpc" + "github.com/coze-dev/coze-loop/backend/modules/observability/domain/component/tenant" + "github.com/coze-dev/coze-loop/backend/modules/observability/domain/component/workspace" "github.com/coze-dev/coze-loop/backend/modules/observability/domain/trace/entity" "github.com/coze-dev/coze-loop/backend/modules/observability/domain/trace/entity/loop_span" "github.com/coze-dev/coze-loop/backend/modules/observability/domain/trace/service" @@ -269,7 +268,7 @@ func (o *OpenAPIApplication) OtelIngestTraces(ctx context.Context, req *openapi. spans := otel.OtelSpansConvertToSendSpans(ctx, workspaceId, otelSpans) - tenantSpanMap := o.unpackTenant(ctx, spans) + tenantSpanMap := o.unpackTenant(ctx, tconv.OtelSpans2LoopSpans(spans)) for ingestTenant := range tenantSpanMap { if e = o.traceService.IngestTraces(ctx, &service.IngestTracesReq{ Tenant: ingestTenant, diff --git a/backend/modules/observability/application/wire.go b/backend/modules/observability/application/wire.go index fd5930fa0..4b0a2a594 100644 --- a/backend/modules/observability/application/wire.go +++ b/backend/modules/observability/application/wire.go @@ -25,9 +25,12 @@ import ( "github.com/coze-dev/coze-loop/backend/kitex_gen/coze/loop/foundation/file/fileservice" "github.com/coze-dev/coze-loop/backend/kitex_gen/coze/loop/foundation/user/userservice" "github.com/coze-dev/coze-loop/backend/modules/observability/domain/component/config" + mq3 "github.com/coze-dev/coze-loop/backend/modules/observability/domain/component/mq" "github.com/coze-dev/coze-loop/backend/modules/observability/domain/component/rpc" "github.com/coze-dev/coze-loop/backend/modules/observability/domain/component/scheduledtask" + "github.com/coze-dev/coze-loop/backend/modules/observability/domain/component/storage" metrics_entity "github.com/coze-dev/coze-loop/backend/modules/observability/domain/metric/entity" + metric_repo "github.com/coze-dev/coze-loop/backend/modules/observability/domain/metric/repo" metric_service "github.com/coze-dev/coze-loop/backend/modules/observability/domain/metric/service" metric_general "github.com/coze-dev/coze-loop/backend/modules/observability/domain/metric/service/metric/general" metric_model "github.com/coze-dev/coze-loop/backend/modules/observability/domain/metric/service/metric/model" @@ -66,6 +69,7 @@ import ( "github.com/coze-dev/coze-loop/backend/modules/observability/infra/rpc/file" "github.com/coze-dev/coze-loop/backend/modules/observability/infra/rpc/tag" "github.com/coze-dev/coze-loop/backend/modules/observability/infra/rpc/user" + obstorage "github.com/coze-dev/coze-loop/backend/modules/observability/infra/storage" "github.com/coze-dev/coze-loop/backend/modules/observability/infra/tenant" "github.com/coze-dev/coze-loop/backend/modules/observability/infra/workspace" "github.com/coze-dev/coze-loop/backend/pkg/conf" @@ -88,9 +92,8 @@ var ( traceDomainSet = wire.NewSet( service.NewTraceServiceImpl, service.NewTraceExportServiceImpl, - obrepo.NewTraceCKRepoImpl, - ckdao.NewSpansCkDaoImpl, - ckdao.NewAnnotationCkDaoImpl, + provideTraceRepo, + obstorage.NewTraceStorageProvider, obmetrics.NewTraceMetricsImpl, obcollector.NewEventCollectorProvider, mq2.NewTraceProducerImpl, @@ -119,9 +122,7 @@ var ( traceIngestionSet = wire.NewSet( NewIngestionApplication, service.NewIngestionServiceImpl, - obrepo.NewTraceCKRepoImpl, - ckdao.NewSpansCkDaoImpl, - ckdao.NewAnnotationCkDaoImpl, + provideTraceRepo, obconfig.NewTraceConfigCenter, NewTraceConfigLoader, NewIngestionCollectorFactory, @@ -146,19 +147,57 @@ var ( metricsSet = wire.NewSet( NewMetricApplication, metric_service.NewMetricsService, - obrepo.NewTraceMetricCKRepoImpl, + provideTraceMetricRepo, tenant.NewTenantProvider, auth.NewAuthProvider, NewTraceConfigLoader, NewTraceProcessorBuilder, obconfig.NewTraceConfigCenter, NewMetricDefinitions, - ckdao.NewSpansCkDaoImpl, - ckdao.NewAnnotationCkDaoImpl, file.NewFileRPCProvider, ) ) +func provideTraceRepo( + traceConfig config.ITraceConfig, + storageProvider storage.IStorageProvider, + spanRedisDao redis2.ISpansRedisDao, + ckProvider ck.Provider, + spanProducer mq3.ISpanProducer, +) (repo.ITraceRepo, error) { + options, err := buildTraceRepoOptions(ckProvider) + if err != nil { + return nil, err + } + return obrepo.NewTraceRepoImpl(traceConfig, storageProvider, spanRedisDao, spanProducer, options...) +} + +func provideTraceMetricRepo( + traceConfig config.ITraceConfig, + storageProvider storage.IStorageProvider, + ckProvider ck.Provider, +) (metric_repo.IMetricRepo, error) { + options, err := buildTraceRepoOptions(ckProvider) + if err != nil { + return nil, err + } + return obrepo.NewTraceMetricCKRepoImpl(traceConfig, storageProvider, options...) +} + +func buildTraceRepoOptions(ckProvider ck.Provider) ([]obrepo.TraceRepoOption, error) { + ckSpanDao, err := ckdao.NewSpansCkDaoImpl(ckProvider) + if err != nil { + return nil, err + } + ckAnnoDao, err := ckdao.NewAnnotationCkDaoImpl(ckProvider) + if err != nil { + return nil, err + } + return []obrepo.TraceRepoOption{ + obrepo.WithTraceStorageDaos(ckdao.TraceStorageTypeCK, ckSpanDao, ckAnnoDao), + }, nil +} + func NewTaskLocker(cmdable redis.Cmdable) lock.ILocker { return lock.NewRedisLockerWithHolder(cmdable, "observability") } @@ -289,7 +328,8 @@ func NewInitTaskProcessor(datasetServiceProvider *service.DatasetServiceAdaptor, evaluationService rpc.IEvaluationRPCAdapter, taskRepo trepo.ITaskRepo, ) *task_processor.TaskProcessor { taskProcessor := task_processor.NewTaskProcessor() - taskProcessor.Register(task_entity.TaskTypeAutoEval, task_processor.NewAutoEvaluteProcessor(0, datasetServiceProvider, evalService, evaluationService, taskRepo)) + taskProcessor.Register(task_entity.TaskTypeAutoEval, task_processor.NewAutoEvaluateProcessor( + 0, datasetServiceProvider, evalService, evaluationService, taskRepo, &task_processor.EvalTargetBuilderImpl{})) return taskProcessor } @@ -350,6 +390,7 @@ func InitOpenAPIApplication( func InitMetricApplication( ckDb ck.Provider, + storageProvider storage.IStorageProvider, configFactory conf.IConfigLoaderFactory, fileClient fileservice.Client, benefit benefit.IBenefitService, @@ -361,6 +402,7 @@ func InitMetricApplication( func InitTraceIngestionApplication( configFactory conf.IConfigLoaderFactory, + storageProvider storage.IStorageProvider, ckDb ck.Provider, mqFactory mq.IFactory, persistentCmdable redis.PersistentCmdable, diff --git a/backend/modules/observability/application/wire_gen.go b/backend/modules/observability/application/wire_gen.go index 47a383e32..a2a8e511d 100644 --- a/backend/modules/observability/application/wire_gen.go +++ b/backend/modules/observability/application/wire_gen.go @@ -25,16 +25,19 @@ import ( "github.com/coze-dev/coze-loop/backend/kitex_gen/coze/loop/foundation/file/fileservice" "github.com/coze-dev/coze-loop/backend/kitex_gen/coze/loop/foundation/user/userservice" config2 "github.com/coze-dev/coze-loop/backend/modules/observability/domain/component/config" + mq2 "github.com/coze-dev/coze-loop/backend/modules/observability/domain/component/mq" "github.com/coze-dev/coze-loop/backend/modules/observability/domain/component/rpc" "github.com/coze-dev/coze-loop/backend/modules/observability/domain/component/scheduledtask" + storage2 "github.com/coze-dev/coze-loop/backend/modules/observability/domain/component/storage" "github.com/coze-dev/coze-loop/backend/modules/observability/domain/metric/entity" + repo3 "github.com/coze-dev/coze-loop/backend/modules/observability/domain/metric/repo" service2 "github.com/coze-dev/coze-loop/backend/modules/observability/domain/metric/service" "github.com/coze-dev/coze-loop/backend/modules/observability/domain/metric/service/metric/general" "github.com/coze-dev/coze-loop/backend/modules/observability/domain/metric/service/metric/model" service4 "github.com/coze-dev/coze-loop/backend/modules/observability/domain/metric/service/metric/service" "github.com/coze-dev/coze-loop/backend/modules/observability/domain/metric/service/metric/tool" entity3 "github.com/coze-dev/coze-loop/backend/modules/observability/domain/task/entity" - repo3 "github.com/coze-dev/coze-loop/backend/modules/observability/domain/task/repo" + repo4 "github.com/coze-dev/coze-loop/backend/modules/observability/domain/task/repo" service3 "github.com/coze-dev/coze-loop/backend/modules/observability/domain/task/service" "github.com/coze-dev/coze-loop/backend/modules/observability/domain/task/service/taskexe/processor" scheduledtask2 "github.com/coze-dev/coze-loop/backend/modules/observability/domain/task/service/taskexe/scheduledtask" @@ -66,6 +69,7 @@ import ( "github.com/coze-dev/coze-loop/backend/modules/observability/infra/rpc/file" "github.com/coze-dev/coze-loop/backend/modules/observability/infra/rpc/tag" "github.com/coze-dev/coze-loop/backend/modules/observability/infra/rpc/user" + "github.com/coze-dev/coze-loop/backend/modules/observability/infra/storage" "github.com/coze-dev/coze-loop/backend/modules/observability/infra/tenant" "github.com/coze-dev/coze-loop/backend/modules/observability/infra/workspace" "github.com/coze-dev/coze-loop/backend/pkg/conf" @@ -75,19 +79,12 @@ import ( // Injectors from wire.go: func InitTraceApplication(db2 db.Provider, ckDb ck.Provider, redis3 redis.Cmdable, persistentCmdable redis.PersistentCmdable, meter metrics.Meter, mqFactory mq.IFactory, configFactory conf.IConfigLoaderFactory, idgen2 idgen.IIDGenerator, fileClient fileservice.Client, benefit2 benefit.IBenefitService, authClient authservice.Client, userClient userservice.Client, evalService evaluatorservice.Client, evalSetService evaluationsetservice.Client, tagService tagservice.Client, datasetService datasetservice.Client) (ITraceApplication, error) { - iSpansDao, err := ck2.NewSpansCkDaoImpl(ckDb) - if err != nil { - return nil, err - } - iAnnotationDao, err := ck2.NewAnnotationCkDaoImpl(ckDb) - if err != nil { - return nil, err - } iConfigLoader, err := NewTraceConfigLoader(configFactory) if err != nil { return nil, err } iTraceConfig := config.NewTraceConfigCenter(iConfigLoader) + iStorageProvider := storage.NewTraceStorageProvider() iSpansRedisDao, err := redis2.NewSpansRedisDaoImpl(persistentCmdable) if err != nil { return nil, err @@ -96,7 +93,7 @@ func InitTraceApplication(db2 db.Provider, ckDb ck.Provider, redis3 redis.Cmdabl if err != nil { return nil, err } - iTraceRepo, err := repo.NewTraceCKRepoImpl(iSpansDao, iAnnotationDao, iTraceConfig, iSpansRedisDao, iSpanProducer) + iTraceRepo, err := provideTraceRepo(iTraceConfig, iStorageProvider, iSpansRedisDao, ckDb, iSpanProducer) if err != nil { return nil, err } @@ -123,7 +120,7 @@ func InitTraceApplication(db2 db.Provider, ckDb ck.Provider, redis3 redis.Cmdabl return nil, err } datasetServiceAdaptor := NewDatasetServiceAdapter(evalSetService, datasetService) - iTraceExportService, err := service.NewTraceExportServiceImpl(iTraceRepo, iTraceConfig, iTraceProducer, iAnnotationProducer, iTraceMetrics, iTenantProvider, datasetServiceAdaptor, traceFilterProcessorBuilder) + iTraceExportService, err := service.NewTraceExportServiceImpl(iTraceRepo, iStorageProvider, iTraceConfig, iTraceProducer, iAnnotationProducer, iTraceMetrics, iTenantProvider, datasetServiceAdaptor, traceFilterProcessorBuilder) if err != nil { return nil, err } @@ -140,19 +137,12 @@ func InitTraceApplication(db2 db.Provider, ckDb ck.Provider, redis3 redis.Cmdabl } func InitOpenAPIApplication(mqFactory mq.IFactory, configFactory conf.IConfigLoaderFactory, fileClient fileservice.Client, ckDb ck.Provider, benefit2 benefit.IBenefitService, limiterFactory limiter.IRateLimiterFactory, authClient authservice.Client, meter metrics.Meter, db2 db.Provider, redis3 redis.Cmdable, idgen2 idgen.IIDGenerator, evalService evaluatorservice.Client, persistentCmdable redis.PersistentCmdable) (IObservabilityOpenAPIApplication, error) { - iSpansDao, err := ck2.NewSpansCkDaoImpl(ckDb) - if err != nil { - return nil, err - } - iAnnotationDao, err := ck2.NewAnnotationCkDaoImpl(ckDb) - if err != nil { - return nil, err - } iConfigLoader, err := NewTraceConfigLoader(configFactory) if err != nil { return nil, err } iTraceConfig := config.NewTraceConfigCenter(iConfigLoader) + iStorageProvider := storage.NewTraceStorageProvider() iSpansRedisDao, err := redis2.NewSpansRedisDaoImpl(persistentCmdable) if err != nil { return nil, err @@ -161,7 +151,7 @@ func InitOpenAPIApplication(mqFactory mq.IFactory, configFactory conf.IConfigLoa if err != nil { return nil, err } - iTraceRepo, err := repo.NewTraceCKRepoImpl(iSpansDao, iAnnotationDao, iTraceConfig, iSpansRedisDao, iSpanProducer) + iTraceRepo, err := provideTraceRepo(iTraceConfig, iStorageProvider, iSpansRedisDao, ckDb, iSpanProducer) if err != nil { return nil, err } @@ -197,21 +187,13 @@ func InitOpenAPIApplication(mqFactory mq.IFactory, configFactory conf.IConfigLoa return iObservabilityOpenAPIApplication, nil } -func InitMetricApplication(ckDb ck.Provider, configFactory conf.IConfigLoaderFactory, fileClient fileservice.Client, benefit2 benefit.IBenefitService, authClient authservice.Client) (IMetricApplication, error) { - iSpansDao, err := ck2.NewSpansCkDaoImpl(ckDb) - if err != nil { - return nil, err - } - iAnnotationDao, err := ck2.NewAnnotationCkDaoImpl(ckDb) - if err != nil { - return nil, err - } +func InitMetricApplication(ckDb ck.Provider, storageProvider storage2.IStorageProvider, configFactory conf.IConfigLoaderFactory, fileClient fileservice.Client, benefit2 benefit.IBenefitService, authClient authservice.Client) (IMetricApplication, error) { iConfigLoader, err := NewTraceConfigLoader(configFactory) if err != nil { return nil, err } iTraceConfig := config.NewTraceConfigCenter(iConfigLoader) - iMetricRepo, err := repo.NewTraceMetricCKRepoImpl(iSpansDao, iAnnotationDao, iTraceConfig) + iMetricRepo, err := provideTraceMetricRepo(iTraceConfig, storageProvider, ckDb) if err != nil { return nil, err } @@ -231,19 +213,11 @@ func InitMetricApplication(ckDb ck.Provider, configFactory conf.IConfigLoaderFac return iMetricApplication, nil } -func InitTraceIngestionApplication(configFactory conf.IConfigLoaderFactory, ckDb ck.Provider, mqFactory mq.IFactory, persistentCmdable redis.PersistentCmdable) (ITraceIngestionApplication, error) { +func InitTraceIngestionApplication(configFactory conf.IConfigLoaderFactory, storageProvider storage2.IStorageProvider, ckDb ck.Provider, mqFactory mq.IFactory, persistentCmdable redis.PersistentCmdable) (ITraceIngestionApplication, error) { iConfigLoader, err := NewTraceConfigLoader(configFactory) if err != nil { return nil, err } - iSpansDao, err := ck2.NewSpansCkDaoImpl(ckDb) - if err != nil { - return nil, err - } - iAnnotationDao, err := ck2.NewAnnotationCkDaoImpl(ckDb) - if err != nil { - return nil, err - } iTraceConfig := config.NewTraceConfigCenter(iConfigLoader) iSpansRedisDao, err := redis2.NewSpansRedisDaoImpl(persistentCmdable) if err != nil { @@ -253,7 +227,7 @@ func InitTraceIngestionApplication(configFactory conf.IConfigLoaderFactory, ckDb if err != nil { return nil, err } - iTraceRepo, err := repo.NewTraceCKRepoImpl(iSpansDao, iAnnotationDao, iTraceConfig, iSpansRedisDao, iSpanProducer) + iTraceRepo, err := provideTraceRepo(iTraceConfig, storageProvider, iSpansRedisDao, ckDb, iSpanProducer) if err != nil { return nil, err } @@ -285,22 +259,16 @@ func InitTaskApplication(db2 db.Provider, idgen2 idgen.IIDGenerator, configFacto iEvaluatorRPCAdapter := evaluator.NewEvaluatorRPCProvider(evalService) iEvaluationRPCAdapter := evaluation.NewEvaluationRPCProvider(exptService) processorTaskProcessor := NewInitTaskProcessor(datasetServiceAdaptor, iEvaluatorRPCAdapter, iEvaluationRPCAdapter, iTaskRepo) + iStorageProvider := storage.NewTraceStorageProvider() + iTenantProvider := tenant.NewTenantProvider(iTraceConfig) iFileProvider := file.NewFileRPCProvider(fileClient) traceFilterProcessorBuilder := NewTraceProcessorBuilder(iTraceConfig, iFileProvider, benefit2) - iTaskService, err := service3.NewTaskServiceImpl(iTaskRepo, idgen2, iBackfillProducer, processorTaskProcessor, traceFilterProcessorBuilder) + iTaskService, err := service3.NewTaskServiceImpl(iTaskRepo, idgen2, iBackfillProducer, processorTaskProcessor, iStorageProvider, iTenantProvider, traceFilterProcessorBuilder) if err != nil { return nil, err } iAuthProvider := auth.NewAuthProvider(authClient) iUserProvider := user.NewUserRPCProvider(userClient) - iSpansDao, err := ck2.NewSpansCkDaoImpl(ckDb) - if err != nil { - return nil, err - } - iAnnotationDao, err := ck2.NewAnnotationCkDaoImpl(ckDb) - if err != nil { - return nil, err - } iSpansRedisDao, err := redis2.NewSpansRedisDaoImpl(persistentCmdable) if err != nil { return nil, err @@ -309,11 +277,10 @@ func InitTaskApplication(db2 db.Provider, idgen2 idgen.IIDGenerator, configFacto if err != nil { return nil, err } - iTraceRepo, err := repo.NewTraceCKRepoImpl(iSpansDao, iAnnotationDao, iTraceConfig, iSpansRedisDao, iSpanProducer) + iTraceRepo, err := provideTraceRepo(iTraceConfig, iStorageProvider, iSpansRedisDao, ckDb, iSpanProducer) if err != nil { return nil, err } - iTenantProvider := tenant.NewTenantProvider(iTraceConfig) iLocker := NewTaskLocker(redis3) iTraceHubService, err := tracehub.NewTraceHubImpl(iTaskRepo, iTraceRepo, iTenantProvider, traceFilterProcessorBuilder, processorTaskProcessor, aid, iBackfillProducer, iLocker, iTraceConfig) if err != nil { @@ -334,14 +301,14 @@ var ( taskDomainSet = wire.NewSet( NewInitTaskProcessor, service3.NewTaskServiceImpl, repo.NewTaskRepoImpl, mysql.NewTaskDaoImpl, redis2.NewTaskDAO, redis2.NewTaskRunDAO, mysql.NewTaskRunDaoImpl, producer.NewBackfillProducerImpl, NewScheduledTask, ) - traceDomainSet = wire.NewSet(service.NewTraceServiceImpl, service.NewTraceExportServiceImpl, repo.NewTraceCKRepoImpl, ck2.NewSpansCkDaoImpl, ck2.NewAnnotationCkDaoImpl, metrics2.NewTraceMetricsImpl, collector.NewEventCollectorProvider, producer.NewTraceProducerImpl, producer.NewAnnotationProducerImpl, producer.NewSpanWithAnnotationProducerImpl, file.NewFileRPCProvider, NewTraceConfigLoader, + traceDomainSet = wire.NewSet(service.NewTraceServiceImpl, service.NewTraceExportServiceImpl, provideTraceRepo, storage.NewTraceStorageProvider, metrics2.NewTraceMetricsImpl, collector.NewEventCollectorProvider, producer.NewTraceProducerImpl, producer.NewAnnotationProducerImpl, producer.NewSpanWithAnnotationProducerImpl, file.NewFileRPCProvider, NewTraceConfigLoader, NewTraceProcessorBuilder, config.NewTraceConfigCenter, tenant.NewTenantProvider, workspace.NewWorkspaceProvider, evaluator.NewEvaluatorRPCProvider, NewDatasetServiceAdapter, redis2.NewSpansRedisDaoImpl, taskDomainSet, ) traceSet = wire.NewSet( NewTraceApplication, repo.NewViewRepoImpl, mysql.NewViewDaoImpl, auth.NewAuthProvider, user.NewUserRPCProvider, tag.NewTagRPCProvider, traceDomainSet, ) traceIngestionSet = wire.NewSet( - NewIngestionApplication, service.NewIngestionServiceImpl, repo.NewTraceCKRepoImpl, ck2.NewSpansCkDaoImpl, ck2.NewAnnotationCkDaoImpl, config.NewTraceConfigCenter, NewTraceConfigLoader, + NewIngestionApplication, service.NewIngestionServiceImpl, provideTraceRepo, config.NewTraceConfigCenter, NewTraceConfigLoader, NewIngestionCollectorFactory, producer.NewSpanWithAnnotationProducerImpl, redis2.NewSpansRedisDaoImpl, ) openApiSet = wire.NewSet( @@ -351,11 +318,49 @@ var ( traceDomainSet, service3.NewTaskCallbackServiceImpl, ) metricsSet = wire.NewSet( - NewMetricApplication, service2.NewMetricsService, repo.NewTraceMetricCKRepoImpl, tenant.NewTenantProvider, auth.NewAuthProvider, NewTraceConfigLoader, - NewTraceProcessorBuilder, config.NewTraceConfigCenter, NewMetricDefinitions, ck2.NewSpansCkDaoImpl, ck2.NewAnnotationCkDaoImpl, file.NewFileRPCProvider, + NewMetricApplication, service2.NewMetricsService, provideTraceMetricRepo, tenant.NewTenantProvider, auth.NewAuthProvider, NewTraceConfigLoader, + NewTraceProcessorBuilder, config.NewTraceConfigCenter, NewMetricDefinitions, file.NewFileRPCProvider, ) ) +func provideTraceRepo( + traceConfig config2.ITraceConfig, + storageProvider storage2.IStorageProvider, + spanRedisDao redis2.ISpansRedisDao, + ckProvider ck.Provider, + spanProducer mq2.ISpanProducer, +) (repo2.ITraceRepo, error) { + options, err := buildTraceRepoOptions(ckProvider) + if err != nil { + return nil, err + } + return repo.NewTraceRepoImpl(traceConfig, storageProvider, spanRedisDao, spanProducer, options...) +} + +func provideTraceMetricRepo( + traceConfig config2.ITraceConfig, + storageProvider storage2.IStorageProvider, + ckProvider ck.Provider, +) (repo3.IMetricRepo, error) { + options, err := buildTraceRepoOptions(ckProvider) + if err != nil { + return nil, err + } + return repo.NewTraceMetricCKRepoImpl(traceConfig, storageProvider, options...) +} + +func buildTraceRepoOptions(ckProvider ck.Provider) ([]repo.TraceRepoOption, error) { + ckSpanDao, err := ck2.NewSpansCkDaoImpl(ckProvider) + if err != nil { + return nil, err + } + ckAnnoDao, err := ck2.NewAnnotationCkDaoImpl(ckProvider) + if err != nil { + return nil, err + } + return []repo.TraceRepoOption{repo.WithTraceStorageDaos(ck2.TraceStorageTypeCK, ckSpanDao, ckAnnoDao)}, nil +} + func NewTaskLocker(cmdable redis.Cmdable) lock.ILocker { return lock.NewRedisLockerWithHolder(cmdable, "observability") } @@ -403,10 +408,11 @@ func NewDatasetServiceAdapter(evalSetService evaluationsetservice.Client, datase } func NewInitTaskProcessor(datasetServiceProvider *service.DatasetServiceAdaptor, evalService rpc.IEvaluatorRPCAdapter, - evaluationService rpc.IEvaluationRPCAdapter, taskRepo repo3.ITaskRepo, + evaluationService rpc.IEvaluationRPCAdapter, taskRepo repo4.ITaskRepo, ) *processor.TaskProcessor { taskProcessor := processor.NewTaskProcessor() - taskProcessor.Register(entity3.TaskTypeAutoEval, processor.NewAutoEvaluteProcessor(0, datasetServiceProvider, evalService, evaluationService, taskRepo)) + taskProcessor.Register(entity3.TaskTypeAutoEval, processor.NewAutoEvaluateProcessor( + 0, datasetServiceProvider, evalService, evaluationService, taskRepo, &processor.EvalTargetBuilderImpl{})) return taskProcessor } @@ -416,7 +422,7 @@ func NewScheduledTask( traceHubService tracehub.ITraceHubService, taskService service3.ITaskService, taskProcessor processor.TaskProcessor, - taskRepo repo3.ITaskRepo, + taskRepo repo4.ITaskRepo, ) []scheduledtask.ScheduledTask { return []scheduledtask.ScheduledTask{scheduledtask2.NewStatusCheckTask(locker, config3, traceHubService, taskService, taskProcessor, taskRepo), scheduledtask2.NewLocalCacheRefreshTask(traceHubService, taskRepo)} } diff --git a/backend/modules/observability/domain/component/storage/mocks/storage_provider.go b/backend/modules/observability/domain/component/storage/mocks/storage_provider.go new file mode 100644 index 000000000..e05492fcb --- /dev/null +++ b/backend/modules/observability/domain/component/storage/mocks/storage_provider.go @@ -0,0 +1,70 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/coze-dev/coze-loop/backend/modules/observability/domain/component/storage (interfaces: IStorageProvider) +// +// Generated by this command: +// +// mockgen -destination=mocks/storage_provider.go -package=mocks . IStorageProvider +// + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + reflect "reflect" + + storage "github.com/coze-dev/coze-loop/backend/modules/observability/domain/component/storage" + gomock "go.uber.org/mock/gomock" +) + +// MockIStorageProvider is a mock of IStorageProvider interface. +type MockIStorageProvider struct { + ctrl *gomock.Controller + recorder *MockIStorageProviderMockRecorder + isgomock struct{} +} + +// MockIStorageProviderMockRecorder is the mock recorder for MockIStorageProvider. +type MockIStorageProviderMockRecorder struct { + mock *MockIStorageProvider +} + +// NewMockIStorageProvider creates a new mock instance. +func NewMockIStorageProvider(ctrl *gomock.Controller) *MockIStorageProvider { + mock := &MockIStorageProvider{ctrl: ctrl} + mock.recorder = &MockIStorageProviderMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockIStorageProvider) EXPECT() *MockIStorageProviderMockRecorder { + return m.recorder +} + +// GetTraceStorage mocks base method. +func (m *MockIStorageProvider) GetTraceStorage(ctx context.Context, workSpaceID string, tenants []string) storage.Storage { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetTraceStorage", ctx, workSpaceID, tenants) + ret0, _ := ret[0].(storage.Storage) + return ret0 +} + +// GetTraceStorage indicates an expected call of GetTraceStorage. +func (mr *MockIStorageProviderMockRecorder) GetTraceStorage(ctx, workSpaceID, tenants any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTraceStorage", reflect.TypeOf((*MockIStorageProvider)(nil).GetTraceStorage), ctx, workSpaceID, tenants) +} + +// PrepareStorageForTask mocks base method. +func (m *MockIStorageProvider) PrepareStorageForTask(ctx context.Context, workspaceID string, tenants []string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "PrepareStorageForTask", ctx, workspaceID, tenants) + ret0, _ := ret[0].(error) + return ret0 +} + +// PrepareStorageForTask indicates an expected call of PrepareStorageForTask. +func (mr *MockIStorageProviderMockRecorder) PrepareStorageForTask(ctx, workspaceID, tenants any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PrepareStorageForTask", reflect.TypeOf((*MockIStorageProvider)(nil).PrepareStorageForTask), ctx, workspaceID, tenants) +} diff --git a/backend/modules/observability/domain/component/storage/storage.go b/backend/modules/observability/domain/component/storage/storage.go new file mode 100644 index 000000000..767fe2168 --- /dev/null +++ b/backend/modules/observability/domain/component/storage/storage.go @@ -0,0 +1,18 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 +package storage + +import ( + "context" +) + +type Storage struct { + StorageName string + StorageConfig map[string]string +} + +//go:generate mockgen -destination=mocks/storage_provider.go -package=mocks . IStorageProvider +type IStorageProvider interface { + GetTraceStorage(ctx context.Context, workSpaceID string, tenants []string) Storage + PrepareStorageForTask(ctx context.Context, workspaceID string, tenants []string) error +} diff --git a/backend/modules/observability/domain/metric/repo/metric.go b/backend/modules/observability/domain/metric/repo/metric.go index 7cc2e290a..fd07b4ff5 100644 --- a/backend/modules/observability/domain/metric/repo/metric.go +++ b/backend/modules/observability/domain/metric/repo/metric.go @@ -11,6 +11,7 @@ import ( ) type GetMetricsParam struct { + WorkSpaceID string Tenants []string Aggregations []*entity.Dimension GroupBys []*entity.Dimension diff --git a/backend/modules/observability/domain/metric/service/metric.go b/backend/modules/observability/domain/metric/service/metric.go index d4525a754..060f3a193 100644 --- a/backend/modules/observability/domain/metric/service/metric.go +++ b/backend/modules/observability/domain/metric/service/metric.go @@ -203,9 +203,10 @@ func (m *MetricsService) buildMetricQuery(ctx context.Context, req *QueryMetrics return nil, err } param := &repo.GetMetricsParam{ - Tenants: tenants, - StartAt: req.StartTime, - EndAt: req.EndTime, + WorkSpaceID: strconv.FormatInt(req.WorkspaceID, 10), + Tenants: tenants, + StartAt: req.StartTime, + EndAt: req.EndTime, } mBuilder := &metricQueryBuilder{ metricNames: req.MetricsNames, diff --git a/backend/modules/observability/domain/task/entity/event.go b/backend/modules/observability/domain/task/entity/event.go index 2fefb720c..36c9fffdc 100644 --- a/backend/modules/observability/domain/task/entity/event.go +++ b/backend/modules/observability/domain/task/entity/event.go @@ -151,15 +151,12 @@ func (s *RawSpan) RawSpanConvertToLoopSpan() *loop_span.Span { LogID: s.LogID, TraceID: s.TraceID, DurationMicros: s.DurationInUs / 1000, - PSM: s.ServerEnv.PSM, CallType: callType, WorkspaceID: spaceID, SpanName: s.SpanName, SpanType: spanType, Method: s.Method, StatusCode: s.StatusCode, - Input: s.SensitiveTags.Input, - Output: s.SensitiveTags.Output, SystemTagsString: systemTagsString, SystemTagsLong: systemTagsLong, SystemTagsDouble: systemTagsDouble, @@ -169,6 +166,13 @@ func (s *RawSpan) RawSpanConvertToLoopSpan() *loop_span.Span { TagsBool: tagsBool, TagsByte: tagsByte, } + if s.ServerEnv != nil { + result.PSM = s.ServerEnv.PSM + } + if s.SensitiveTags != nil { + result.Input = s.SensitiveTags.Input + result.Output = s.SensitiveTags.Output + } return result } diff --git a/backend/modules/observability/domain/task/service/task_callback.go b/backend/modules/observability/domain/task/service/task_callback.go index 7aaf127e8..9cc844cb1 100644 --- a/backend/modules/observability/domain/task/service/task_callback.go +++ b/backend/modules/observability/domain/task/service/task_callback.go @@ -60,7 +60,15 @@ func NewTaskCallbackServiceImpl( func (t *TaskCallbackServiceImpl) AutoEvalCallback(ctx context.Context, event *entity.AutoEvalEvent) error { for _, turn := range event.TurnEvalResults { workspaceIDStr, workspaceID := turn.GetWorkspaceIDFromExt() - tenants, err := t.tenantProvider.GetTenantsByPlatformType(ctx, loop_span.PlatformType("callback_all")) + task, err := t.taskRepo.GetTask(ctx, turn.GetTaskIDFromExt(), nil, nil) + if err != nil { + return err + } + platformType := loop_span.PlatformType("callback_all") + if task != nil && task.SpanFilter != nil { + platformType = task.SpanFilter.PlatformType + } + tenants, err := t.tenantProvider.GetTenantsByPlatformType(ctx, platformType) if err != nil { return err } @@ -115,6 +123,7 @@ func (t *TaskCallbackServiceImpl) AutoEvalCallback(ctx context.Context, event *e } err = t.traceRepo.InsertAnnotations(ctx, &tracerepo.InsertAnnotationParam{ + WorkSpaceID: workspaceIDStr, Tenant: span.GetTenant(), TTL: span.GetTTL(ctx), Span: span, @@ -132,7 +141,15 @@ func (t *TaskCallbackServiceImpl) AutoEvalCorrection(ctx context.Context, event if workspaceID == 0 { return fmt.Errorf("workspace_id is empty") } - tenants, err := t.tenantProvider.GetTenantsByPlatformType(ctx, loop_span.PlatformType("callback_all")) + task, err := t.taskRepo.GetTask(ctx, event.GetTaskIDFromExt(), nil, nil) + if err != nil { + return err + } + platformType := loop_span.PlatformType("callback_all") + if task != nil && task.SpanFilter != nil { + platformType = task.SpanFilter.PlatformType + } + tenants, err := t.tenantProvider.GetTenantsByPlatformType(ctx, platformType) if err != nil { return err } @@ -174,6 +191,7 @@ func (t *TaskCallbackServiceImpl) AutoEvalCorrection(ctx context.Context, event // Then synchronize the observability data param := &tracerepo.InsertAnnotationParam{ + WorkSpaceID: workspaceIDStr, Tenant: span.GetTenant(), TTL: span.GetTTL(ctx), Span: span, @@ -220,7 +238,8 @@ func (t *TaskCallbackServiceImpl) getSpan(ctx context.Context, tenants []string, // todo 目前可能有不同tenant在不同存储中，需要上层多次查询。后续逻辑需要下沉到repo中。 for _, tenant := range tenants { res, err := t.traceRepo.ListSpans(ctx, &tracerepo.ListSpansParam{ - Tenants: []string{tenant}, + WorkSpaceID: workspaceId, + Tenants: []string{tenant}, Filters: &loop_span.FilterFields{ FilterFields: filterFields, }, diff --git a/backend/modules/observability/domain/task/service/task_callback_test.go b/backend/modules/observability/domain/task/service/task_callback_test.go index 18ef2bdee..4f245366e 100755 --- a/backend/modules/observability/domain/task/service/task_callback_test.go +++ b/backend/modules/observability/domain/task/service/task_callback_test.go @@ -47,6 +47,12 @@ func TestTaskCallbackServiceImpl_CallBackSuccess(t *testing.T) { mockTenant.EXPECT().GetTenantsByPlatformType(gomock.Any(), gomock.Any()).Return([]string{"tenant"}, nil).AnyTimes() mockBenefit.EXPECT().CheckTraceBenefit(gomock.Any(), gomock.Any()).Return(&benefit.CheckTraceBenefitResult{StorageDuration: 1}, nil).AnyTimes() mockConfig.EXPECT().GetTraceDataMaxDurationDay(gomock.Any(), gomock.Any()).Return(int64(7)).AnyTimes() + mockTaskRepo.EXPECT().GetTask(gomock.Any(), int64(101), gomock.Any(), gomock.Any()).Return(&entity.ObservabilityTask{ + ID: 101, + SpanFilter: &entity.SpanFilterFields{ + PlatformType: loop_span.PlatformType("callback_all"), + }, + }, nil).AnyTimes() now := time.Now() span := &loop_span.Span{ @@ -102,12 +108,14 @@ func TestTraceHubServiceImpl_CallBackSpanNotFound(t *testing.T) { mockBenefit := benefit_mocks.NewMockIBenefitService(ctrl) mockTenant := tenant_mocks.NewMockITenantProvider(ctrl) mockTraceRepo := trace_repo_mocks.NewMockITraceRepo(ctrl) + mockTaskRepo := repo_mocks.NewMockITaskRepo(ctrl) mockConfig := config_mocks.NewMockITraceConfig(ctrl) impl := &TaskCallbackServiceImpl{ benefitSvc: mockBenefit, tenantProvider: mockTenant, traceRepo: mockTraceRepo, + taskRepo: mockTaskRepo, config: mockConfig, } @@ -115,6 +123,13 @@ func TestTraceHubServiceImpl_CallBackSpanNotFound(t *testing.T) { mockBenefit.EXPECT().CheckTraceBenefit(gomock.Any(), gomock.Any()).Return(&benefit.CheckTraceBenefitResult{StorageDuration: 1}, nil).AnyTimes() mockConfig.EXPECT().GetTraceDataMaxDurationDay(gomock.Any(), gomock.Any()).Return(int64(7)).AnyTimes() mockTraceRepo.EXPECT().ListSpans(gomock.Any(), gomock.AssignableToTypeOf(&repo.ListSpansParam{})).Return(&repo.ListSpansResult{}, nil).AnyTimes() + // 添加缺失的GetTask期望 + mockTaskRepo.EXPECT().GetTask(gomock.Any(), int64(101), gomock.Any(), gomock.Any()).Return(&entity.ObservabilityTask{ + ID: 101, + SpanFilter: &entity.SpanFilterFields{ + PlatformType: loop_span.PlatformType("callback_all"), + }, + }, nil).AnyTimes() event := &entity.AutoEvalEvent{ TurnEvalResults: []*entity.OnlineExptTurnEvalResult{ diff --git a/backend/modules/observability/domain/task/service/task_service.go b/backend/modules/observability/domain/task/service/task_service.go index f743a392e..7b0134fd9 100644 --- a/backend/modules/observability/domain/task/service/task_service.go +++ b/backend/modules/observability/domain/task/service/task_service.go @@ -6,6 +6,8 @@ package service import ( "context" "fmt" + "github.com/coze-dev/coze-loop/backend/modules/observability/domain/component/storage" + "github.com/coze-dev/coze-loop/backend/modules/observability/domain/component/tenant" "strconv" "time" @@ -83,6 +85,8 @@ func NewTaskServiceImpl( idGenerator idgen.IIDGenerator, backfillProducer mq.IBackfillProducer, taskProcessor *processor.TaskProcessor, + storageProvider storage.IStorageProvider, + tenantProvider tenant.ITenantProvider, buildHelper traceservice.TraceFilterProcessorBuilder, ) (ITaskService, error) { return &TaskServiceImpl{ @@ -90,6 +94,8 @@ func NewTaskServiceImpl( idGenerator: idGenerator, backfillProducer: backfillProducer, taskProcessor: *taskProcessor, + storageProvider: storageProvider, + tenantProvider: tenantProvider, buildHelper: buildHelper, }, nil } @@ -99,10 +105,22 @@ type TaskServiceImpl struct { idGenerator idgen.IIDGenerator backfillProducer mq.IBackfillProducer taskProcessor processor.TaskProcessor + storageProvider storage.IStorageProvider + tenantProvider tenant.ITenantProvider buildHelper traceservice.TraceFilterProcessorBuilder } func (t *TaskServiceImpl) CreateTask(ctx context.Context, req *CreateTaskReq) (resp *CreateTaskResp, err error) { + // storage准备 + tenants, err := t.tenantProvider.GetTenantsByPlatformType(ctx, req.Task.SpanFilter.PlatformType) + if err != nil { + return nil, err + } + if err = t.storageProvider.PrepareStorageForTask(ctx, strconv.FormatInt(req.Task.WorkspaceID, 10), tenants); err != nil { + logs.CtxError(ctx, "PrepareStorageForTask err:%v", err) + return nil, err + } + taskDO := req.Task // 校验task name是否存在 checkResp, err := t.CheckTaskName(ctx, &CheckTaskNameReq{ @@ -133,6 +151,7 @@ func (t *TaskServiceImpl) CreateTask(ctx context.Context, req *CreateTaskReq) (r if err != nil { return nil, err } + // 创建任务的数据准备 // 数据回流任务——创建/更新输出数据集 // 自动评测历史回溯——创建空壳子 @@ -364,6 +383,8 @@ func (t *TaskServiceImpl) SendBackfillMessage(ctx context.Context, event *entity if t.backfillProducer == nil { return errorx.NewByCode(obErrorx.CommonInternalErrorCode, errorx.WithExtraMsg("backfill producer not initialized")) } + // todo ppe test + ctx = context.WithValue(ctx, "K_ENV", "ppe_6788399583") return t.backfillProducer.SendBackfill(ctx, event) } diff --git a/backend/modules/observability/domain/task/service/task_service_test.go b/backend/modules/observability/domain/task/service/task_service_test.go index 36cb002d4..a4022a99c 100755 --- a/backend/modules/observability/domain/task/service/task_service_test.go +++ b/backend/modules/observability/domain/task/service/task_service_test.go @@ -15,6 +15,7 @@ import ( "github.com/coze-dev/coze-loop/backend/infra/middleware/session" componentmq "github.com/coze-dev/coze-loop/backend/modules/observability/domain/component/mq" + "github.com/coze-dev/coze-loop/backend/modules/observability/domain/component/storage" "github.com/coze-dev/coze-loop/backend/modules/observability/domain/task/entity" taskrepo "github.com/coze-dev/coze-loop/backend/modules/observability/domain/task/repo" repomocks "github.com/coze-dev/coze-loop/backend/modules/observability/domain/task/repo/mocks" @@ -23,6 +24,7 @@ import ( "github.com/coze-dev/coze-loop/backend/modules/observability/domain/trace/entity/loop_span" "github.com/coze-dev/coze-loop/backend/modules/observability/domain/trace/service/trace/span_filter" "github.com/coze-dev/coze-loop/backend/modules/observability/domain/trace/service/trace/span_processor" + "github.com/coze-dev/coze-loop/backend/modules/observability/infra/repo/dao" obErrorx "github.com/coze-dev/coze-loop/backend/modules/observability/pkg/errno" "github.com/coze-dev/coze-loop/backend/pkg/errorx" ) @@ -125,13 +127,19 @@ func (s *stubBackfillProducer) SendBackfill(ctx context.Context, message *entity return s.err } -func newTaskServiceWithProcessor(t *testing.T, repo taskrepo.ITaskRepo, backfill componentmq.IBackfillProducer, proc taskexe.Processor, taskType entity.TaskType) *TaskServiceImpl { +func newTaskServiceWithProcessor(t *testing.T, repo taskrepo.ITaskRepo, backfill componentmq.IBackfillProducer, proc taskexe.Processor, taskType entity.TaskType) ITaskService { t.Helper() tp := processor.NewTaskProcessor() tp.Register(taskType, proc) - service, err := NewTaskServiceImpl(repo, nil, backfill, tp, &stubTraceFilterBuilder{}) + // 创建mock依赖 + idGenerator := &stubIDGenerator{} + storageProvider := &stubStorageProvider{} + tenantProvider := &stubTenantProvider{} + buildHelper := &stubTraceFilterBuilder{} + + service, err := NewTaskServiceImpl(repo, idGenerator, backfill, tp, storageProvider, tenantProvider, buildHelper) assert.NoError(t, err) - return service.(*TaskServiceImpl) + return service } func TestTaskServiceImpl_CreateTask(t *testing.T) { @@ -574,7 +582,17 @@ func TestTaskServiceImpl_CheckTaskName(t *testing.T) { repoMock := repomocks.NewMockITaskRepo(ctrl) repoMock.EXPECT().ListTasks(gomock.Any(), gomock.Any()).Return(nil, int64(0), errors.New("repo fail")) - svc := &TaskServiceImpl{TaskRepo: repoMock} + // 使用构造函数创建服务 + svc, err := NewTaskServiceImpl( + repoMock, + &stubIDGenerator{}, + nil, + processor.NewTaskProcessor(), + &stubStorageProvider{}, + &stubTenantProvider{}, + &stubTraceFilterBuilder{}, + ) + assert.NoError(t, err) resp, err := svc.CheckTaskName(context.Background(), &CheckTaskNameReq{WorkspaceID: 1, Name: "task"}) assert.Nil(t, resp) assert.EqualError(t, err, "repo fail") @@ -588,7 +606,17 @@ func TestTaskServiceImpl_CheckTaskName(t *testing.T) { repoMock := repomocks.NewMockITaskRepo(ctrl) repoMock.EXPECT().ListTasks(gomock.Any(), gomock.Any()).Return([]*entity.ObservabilityTask{{}}, int64(1), nil) - svc := &TaskServiceImpl{TaskRepo: repoMock} + // 使用构造函数创建服务 + svc, err := NewTaskServiceImpl( + repoMock, + &stubIDGenerator{}, + nil, + processor.NewTaskProcessor(), + &stubStorageProvider{}, + &stubTenantProvider{}, + &stubTraceFilterBuilder{}, + ) + assert.NoError(t, err) resp, err := svc.CheckTaskName(context.Background(), &CheckTaskNameReq{WorkspaceID: 1, Name: "task"}) assert.NoError(t, err) if assert.NotNil(t, resp) { @@ -604,7 +632,17 @@ func TestTaskServiceImpl_CheckTaskName(t *testing.T) { repoMock := repomocks.NewMockITaskRepo(ctrl) repoMock.EXPECT().ListTasks(gomock.Any(), gomock.Any()).Return(nil, int64(0), nil) - svc := &TaskServiceImpl{TaskRepo: repoMock} + // 使用构造函数创建服务 + svc, err := NewTaskServiceImpl( + repoMock, + &stubIDGenerator{}, + nil, + processor.NewTaskProcessor(), + &stubStorageProvider{}, + &stubTenantProvider{}, + &stubTraceFilterBuilder{}, + ) + assert.NoError(t, err) resp, err := svc.CheckTaskName(context.Background(), &CheckTaskNameReq{WorkspaceID: 1, Name: "task"}) assert.NoError(t, err) if assert.NotNil(t, resp) { @@ -615,8 +653,18 @@ func TestTaskServiceImpl_CheckTaskName(t *testing.T) { func TestTaskServiceImpl_sendBackfillMessage(t *testing.T) { t.Run("producer nil", func(t *testing.T) { - svc := &TaskServiceImpl{} - err := svc.SendBackfillMessage(context.Background(), &entity.BackFillEvent{}) + // 创建一个没有backfillProducer的服务 + service, err := NewTaskServiceImpl( + repomocks.NewMockITaskRepo(gomock.NewController(t)), + &stubIDGenerator{}, + nil, // backfillProducer为nil + processor.NewTaskProcessor(), + &stubStorageProvider{}, + &stubTenantProvider{}, + &stubTraceFilterBuilder{}, + ) + assert.NoError(t, err) + err = service.SendBackfillMessage(context.Background(), &entity.BackFillEvent{}) statusErr, ok := errorx.FromStatusError(err) if assert.True(t, ok) { assert.EqualValues(t, obErrorx.CommonInternalErrorCode, statusErr.Code()) @@ -625,8 +673,18 @@ func TestTaskServiceImpl_sendBackfillMessage(t *testing.T) { t.Run("success", func(t *testing.T) { ch := make(chan *entity.BackFillEvent, 1) - svc := &TaskServiceImpl{backfillProducer: &stubBackfillProducer{ch: ch}} - err := svc.SendBackfillMessage(context.Background(), &entity.BackFillEvent{TaskID: 1}) + backfillProducer := &stubBackfillProducer{ch: ch} + service, err := NewTaskServiceImpl( + repomocks.NewMockITaskRepo(gomock.NewController(t)), + &stubIDGenerator{}, + backfillProducer, + processor.NewTaskProcessor(), + &stubStorageProvider{}, + &stubTenantProvider{}, + &stubTraceFilterBuilder{}, + ) + assert.NoError(t, err) + err = service.SendBackfillMessage(context.Background(), &entity.BackFillEvent{TaskID: 1}) assert.NoError(t, err) select { case event := <-ch: @@ -636,3 +694,53 @@ func TestTaskServiceImpl_sendBackfillMessage(t *testing.T) { } }) } + +// 添加缺失的stub实现 +type stubIDGenerator struct{} + +func (s *stubIDGenerator) GenID(ctx context.Context) (int64, error) { + return 1001, nil +} + +func (s *stubIDGenerator) GenMultiIDs(ctx context.Context, counts int) ([]int64, error) { + ids := make([]int64, counts) + for i := 0; i < counts; i++ { + ids[i] = int64(1001 + i) + } + return ids, nil +} + +type stubStorageProvider struct{} + +func (s *stubStorageProvider) GetTraceStorage(ctx context.Context, workSpaceID string, tenants []string) storage.Storage { + return storage.Storage{ + StorageName: "ck", + StorageConfig: map[string]string{}, + } +} + +func (s *stubStorageProvider) PrepareStorageForTask(ctx context.Context, workspaceID string, tenants []string) error { + return nil +} + +func (s *stubStorageProvider) GetSpanDao(tenant string) dao.ISpansDao { + return nil +} + +func (s *stubStorageProvider) GetAnnotationDao(tenant string) dao.IAnnotationDao { + return nil +} + +type stubTenantProvider struct{} + +func (s *stubTenantProvider) GetIngestTenant(ctx context.Context, spans []*loop_span.Span) string { + return "test-tenant" +} + +func (s *stubTenantProvider) GetOAPIQueryTenants(ctx context.Context, platformType loop_span.PlatformType) []string { + return []string{"test-tenant"} +} + +func (s *stubTenantProvider) GetTenantsByPlatformType(ctx context.Context, platformType loop_span.PlatformType) ([]string, error) { + return []string{"test-tenant"}, nil +} diff --git a/backend/modules/observability/domain/task/service/taskexe/processor/auto_evaluate.go b/backend/modules/observability/domain/task/service/taskexe/processor/auto_evaluate.go index 3edbd6450..d91f5c2a1 100644 --- a/backend/modules/observability/domain/task/service/taskexe/processor/auto_evaluate.go +++ b/backend/modules/observability/domain/task/service/taskexe/processor/auto_evaluate.go @@ -13,9 +13,7 @@ import ( "github.com/coze-dev/coze-loop/backend/infra/middleware/session" "github.com/coze-dev/coze-loop/backend/kitex_gen/coze/loop/evaluation/domain/common" "github.com/coze-dev/coze-loop/backend/kitex_gen/coze/loop/evaluation/domain/eval_set" - eval_target_d "github.com/coze-dev/coze-loop/backend/kitex_gen/coze/loop/evaluation/domain/eval_target" "github.com/coze-dev/coze-loop/backend/kitex_gen/coze/loop/evaluation/domain/expt" - "github.com/coze-dev/coze-loop/backend/kitex_gen/coze/loop/evaluation/eval_target" dataset0 "github.com/coze-dev/coze-loop/backend/kitex_gen/coze/loop/observability/domain/dataset" "github.com/coze-dev/coze-loop/backend/kitex_gen/coze/loop/observability/domain/task" tconv "github.com/coze-dev/coze-loop/backend/modules/observability/application/convertor/task" @@ -34,33 +32,36 @@ import ( "github.com/spf13/cast" ) -var _ taskexe.Processor = (*AutoEvaluteProcessor)(nil) +var _ taskexe.Processor = (*AutoEvaluateProcessor)(nil) -type AutoEvaluteProcessor struct { +type AutoEvaluateProcessor struct { evalSvc rpc.IEvaluatorRPCAdapter evaluationSvc rpc.IEvaluationRPCAdapter datasetServiceAdaptor *service.DatasetServiceAdaptor taskRepo repo.ITaskRepo aid int32 + evalTargetBuilder EvalTargetBuilder } -func NewAutoEvaluteProcessor( +func NewAutoEvaluateProcessor( aid int32, datasetServiceProvider *service.DatasetServiceAdaptor, evalService rpc.IEvaluatorRPCAdapter, evaluationService rpc.IEvaluationRPCAdapter, taskRepo repo.ITaskRepo, -) *AutoEvaluteProcessor { - return &AutoEvaluteProcessor{ + evalTargetBuilder EvalTargetBuilder, +) *AutoEvaluateProcessor { + return &AutoEvaluateProcessor{ datasetServiceAdaptor: datasetServiceProvider, evalSvc: evalService, evaluationSvc: evaluationService, taskRepo: taskRepo, aid: aid, + evalTargetBuilder: evalTargetBuilder, } } -func (p *AutoEvaluteProcessor) ValidateConfig(ctx context.Context, config any) error { +func (p *AutoEvaluateProcessor) ValidateConfig(ctx context.Context, config any) error { cfg, ok := config.(*task_entity.ObservabilityTask) if !ok { return errorx.NewByCode(obErrorx.CommonInvalidParamCode) @@ -96,7 +97,7 @@ func (p *AutoEvaluteProcessor) ValidateConfig(ctx context.Context, config any) e return nil } -func (p *AutoEvaluteProcessor) Invoke(ctx context.Context, trigger *taskexe.Trigger) error { +func (p *AutoEvaluateProcessor) Invoke(ctx context.Context, trigger *taskexe.Trigger) error { taskRun := tconv.TaskRunDO2DTO(ctx, trigger.TaskRun, nil) if taskRun.GetTaskRunConfig().GetAutoEvaluateRunConfig() == nil { return nil @@ -109,7 +110,7 @@ func (p *AutoEvaluteProcessor) Invoke(ctx context.Context, trigger *taskexe.Trig } turns := buildItems(ctx, []*loop_span.Span{trigger.Span}, mapping, taskRun.GetTaskRunConfig().GetAutoEvaluateRunConfig().GetSchema(), strconv.FormatInt(taskRun.ID, 10)) if len(turns) == 0 { - logs.CtxInfo(ctx, "[task-debug] AutoEvaluteProcessor Invoke, turns is empty") + logs.CtxInfo(ctx, "[task-debug] AutoEvaluateProcessor Invoke, turns is empty") return nil } taskTTL := trigger.Task.GetTaskttl() @@ -119,7 +120,7 @@ func (p *AutoEvaluteProcessor) Invoke(ctx context.Context, trigger *taskexe.Trig taskRunCount, _ := p.taskRepo.GetTaskRunCount(ctx, trigger.Task.ID, taskRun.ID) if (trigger.Task.Sampler.CycleCount != 0 && taskRunCount > trigger.Task.Sampler.CycleCount) || (taskCount > trigger.Task.Sampler.SampleSize) { - logs.CtxInfo(ctx, "[task-debug] AutoEvaluteProcessor Invoke, subCount:%v,taskCount:%v", taskRunCount, taskCount) + logs.CtxInfo(ctx, "[task-debug] AutoEvaluateProcessor Invoke, subCount:%v,taskCount:%v", taskRunCount, taskCount) _ = p.taskRepo.DecrTaskCount(ctx, trigger.Task.ID, taskTTL) _ = p.taskRepo.DecrTaskRunCount(ctx, trigger.Task.ID, taskRun.ID, taskTTL) return nil @@ -156,7 +157,7 @@ func (p *AutoEvaluteProcessor) Invoke(ctx context.Context, trigger *taskexe.Trig return nil } -func (p *AutoEvaluteProcessor) OnTaskCreated(ctx context.Context, currentTask *task_entity.ObservabilityTask) error { +func (p *AutoEvaluateProcessor) OnTaskCreated(ctx context.Context, currentTask *task_entity.ObservabilityTask) error { taskRuns, err := p.taskRepo.GetBackfillTaskRun(ctx, nil, currentTask.ID) if err != nil { logs.CtxError(ctx, "GetBackfillTaskRun failed, taskID:%d, err:%v", currentTask.ID, err) @@ -200,7 +201,7 @@ func (p *AutoEvaluteProcessor) OnTaskCreated(ctx context.Context, currentTask *t return nil } -func (p *AutoEvaluteProcessor) OnTaskUpdated(ctx context.Context, currentTask *task_entity.ObservabilityTask, taskOp task_entity.TaskStatus) error { +func (p *AutoEvaluateProcessor) OnTaskUpdated(ctx context.Context, currentTask *task_entity.ObservabilityTask, taskOp task_entity.TaskStatus) error { switch taskOp { case task_entity.TaskStatusSuccess: if currentTask.TaskStatus != task_entity.TaskStatusDisabled { @@ -230,7 +231,7 @@ func (p *AutoEvaluteProcessor) OnTaskUpdated(ctx context.Context, currentTask *t return nil } -func (p *AutoEvaluteProcessor) OnTaskFinished(ctx context.Context, param taskexe.OnTaskFinishedReq) error { +func (p *AutoEvaluateProcessor) OnTaskFinished(ctx context.Context, param taskexe.OnTaskFinishedReq) error { err := p.OnTaskRunFinished(ctx, taskexe.OnTaskRunFinishedReq{ Task: param.Task, TaskRun: param.TaskRun, @@ -260,7 +261,7 @@ const ( BackFillI18N = "BackFill" ) -func (p *AutoEvaluteProcessor) OnTaskRunCreated(ctx context.Context, param taskexe.OnTaskRunCreatedReq) error { +func (p *AutoEvaluateProcessor) OnTaskRunCreated(ctx context.Context, param taskexe.OnTaskRunCreatedReq) error { currentTask := param.CurrentTask ctx = session.WithCtxUser(ctx, &session.User{ID: currentTask.CreatedBy}) sessionInfo := p.getSession(ctx, currentTask) @@ -327,7 +328,7 @@ func (p *AutoEvaluteProcessor) OnTaskRunCreated(ctx context.Context, param taske logs.CtxError(ctx, "CreateDataset failed, workspace_id=%d, err=%#v", currentTask.WorkspaceID, err) return err } - logs.CtxInfo(ctx, "[auto_task] AutoEvaluteProcessor OnChangeProcessor, datasetID:%d", datasetID) + logs.CtxInfo(ctx, "[auto_task] AutoEvaluateProcessor OnChangeProcessor, datasetID:%d", datasetID) // Step 2: create experiment maxAliveTime := param.RunEndAt - param.RunStartAt submitExperimentReq := rpc.SubmitExperimentReq{ @@ -341,15 +342,12 @@ func (p *AutoEvaluteProcessor) OnTaskRunCreated(ctx context.Context, param taske TargetFieldMapping: &expt.TargetFieldMapping{ FromEvalSet: []*expt.FieldMapping{}, }, - CreateEvalTargetParam: &eval_target.CreateEvalTargetParam{ - SourceTargetID: gptr.Of(cast.ToString(currentTask.ID)), - EvalTargetType: gptr.Of(eval_target_d.EvalTargetType_Trace), - }, - ExptType: gptr.Of(expt.ExptType_Online), - MaxAliveTime: gptr.Of(maxAliveTime), - SourceType: gptr.Of(expt.SourceType_AutoTask), - SourceID: gptr.Of(cast.ToString(currentTask.ID)), - Session: sessionInfo, + CreateEvalTargetParam: p.evalTargetBuilder.Build(ctx, currentTask), + ExptType: gptr.Of(expt.ExptType_Online), + MaxAliveTime: gptr.Of(maxAliveTime), + SourceType: gptr.Of(expt.SourceType_AutoTask), + SourceID: gptr.Of(cast.ToString(currentTask.ID)), + Session: sessionInfo, } logs.CtxInfo(ctx, "[auto_task] SubmitExperiment:%+v", submitExperimentReq) exptID, exptRunID, err := p.evaluationSvc.SubmitExperiment(ctx, &submitExperimentReq) @@ -357,7 +355,7 @@ func (p *AutoEvaluteProcessor) OnTaskRunCreated(ctx context.Context, param taske logs.CtxError(ctx, "SubmitExperiment failed, workspace_id=%d, err=%#v", currentTask.WorkspaceID, err) return err } - logs.CtxInfo(ctx, "[auto_task] AutoEvaluteProcessor OnChangeProcessor, exptID:%d, exptRunID:%d", exptID, exptRunID) + logs.CtxInfo(ctx, "[auto_task] AutoEvaluateProcessor OnChangeProcessor, exptID:%d, exptRunID:%d", exptID, exptRunID) evaluationSetConfig, err := p.datasetServiceAdaptor.GetDatasetProvider(category).GetDataset(ctx, currentTask.WorkspaceID, datasetID, category) if err != nil { @@ -398,7 +396,7 @@ func (p *AutoEvaluteProcessor) OnTaskRunCreated(ctx context.Context, param taske return nil } -func (p *AutoEvaluteProcessor) OnTaskRunFinished(ctx context.Context, param taskexe.OnTaskRunFinishedReq) error { +func (p *AutoEvaluateProcessor) OnTaskRunFinished(ctx context.Context, param taskexe.OnTaskRunFinishedReq) error { if param.TaskRun == nil || param.TaskRun.TaskRunConfig == nil || param.TaskRun.TaskRunConfig.AutoEvaluateRunConfig == nil { return nil } @@ -423,14 +421,14 @@ func (p *AutoEvaluteProcessor) OnTaskRunFinished(ctx context.Context, param task return nil } -func (p *AutoEvaluteProcessor) getSession(ctx context.Context, task *task_entity.ObservabilityTask) *common.Session { +func (p *AutoEvaluateProcessor) getSession(ctx context.Context, task *task_entity.ObservabilityTask) *common.Session { userIDStr := session.UserIDInCtxOrEmpty(ctx) if userIDStr == "" { userIDStr = task.CreatedBy } userID, err := strconv.ParseInt(userIDStr, 10, 64) if err != nil { - logs.CtxError(ctx, "[task-debug] AutoEvaluteProcessor OnChangeProcessor, ParseInt err:%v", err) + logs.CtxError(ctx, "[task-debug] AutoEvaluateProcessor OnChangeProcessor, ParseInt err:%v", err) } return &common.Session{ UserID: gptr.Of(userID), diff --git a/backend/modules/observability/domain/task/service/taskexe/processor/auto_evaluate_test.go b/backend/modules/observability/domain/task/service/taskexe/processor/auto_evaluate_test.go index b49f56368..85a40cd27 100755 --- a/backend/modules/observability/domain/task/service/taskexe/processor/auto_evaluate_test.go +++ b/backend/modules/observability/domain/task/service/taskexe/processor/auto_evaluate_test.go @@ -143,6 +143,18 @@ func buildTestTask(t *testing.T) *taskentity.ObservabilityTask { CycleInterval: 1, CycleTimeUnit: taskentity.TimeUnitDay, }, + SpanFilter: &taskentity.SpanFilterFields{ + PlatformType: loop_span.PlatformVeAgentKit, + Filters: loop_span.FilterFields{ + FilterFields: []*loop_span.FilterField{ + { + FieldName: "cozeloop_agent_runtime_id", + QueryType: gptr.Of(loop_span.QueryTypeEnumIn), + Values: []string{"test-agent-id"}, + }, + }, + }, + }, TaskConfig: &taskentity.TaskConfig{ AutoEvaluateConfigs: []*taskentity.AutoEvaluateConfig{ { @@ -206,7 +218,7 @@ func makeSchemaJSON(t *testing.T, fieldName string, contentType common.ContentTy return string(bytes) } -func TestAutoEvaluteProcessor_ValidateConfig(t *testing.T) { +func TestAutoEvaluateProcessor_ValidateConfig(t *testing.T) { t.Parallel() ctx := context.Background() @@ -293,7 +305,7 @@ func TestAutoEvaluteProcessor_ValidateConfig(t *testing.T) { for _, tt := range cases { caseItem := tt t.Run(caseItem.name, func(t *testing.T) { - proc := &AutoEvaluteProcessor{evalSvc: caseItem.adapter} + proc := &AutoEvaluateProcessor{evalSvc: caseItem.adapter} if caseItem.adapter == nil { proc.evalSvc = &fakeEvaluatorAdapter{} } @@ -303,7 +315,7 @@ func TestAutoEvaluteProcessor_ValidateConfig(t *testing.T) { } } -func TestAutoEvaluteProcessor_Invoke(t *testing.T) { +func TestAutoEvaluateProcessor_Invoke(t *testing.T) { t.Parallel() textSchema := makeSchemaJSON(t, "field_1", common.ContentTypeText) @@ -335,7 +347,7 @@ func TestAutoEvaluteProcessor_Invoke(t *testing.T) { repoMock := repomocks.NewMockITaskRepo(ctrl) repoAdapter := &taskRepoMockAdapter{MockITaskRepo: repoMock} - proc := &AutoEvaluteProcessor{ + proc := &AutoEvaluateProcessor{ evaluationSvc: &fakeEvaluationAdapter{}, taskRepo: repoAdapter, } @@ -361,7 +373,7 @@ func TestAutoEvaluteProcessor_Invoke(t *testing.T) { repoMock.EXPECT().DecrTaskCount(gomock.Any(), taskObj.ID, gomock.Any()).Return(nil) repoMock.EXPECT().DecrTaskRunCount(gomock.Any(), taskObj.ID, trigger.TaskRun.ID, gomock.Any()).Return(nil) - proc := &AutoEvaluteProcessor{ + proc := &AutoEvaluateProcessor{ evaluationSvc: &fakeEvaluationAdapter{}, taskRepo: repoAdapter, } @@ -389,7 +401,7 @@ func TestAutoEvaluteProcessor_Invoke(t *testing.T) { eval := &fakeEvaluationAdapter{} eval.invokeResp.err = errors.New("invoke fail") - proc := &AutoEvaluteProcessor{ + proc := &AutoEvaluateProcessor{ evaluationSvc: eval, taskRepo: repoAdapter, } @@ -416,7 +428,7 @@ func TestAutoEvaluteProcessor_Invoke(t *testing.T) { repoMock.EXPECT().DecrTaskCount(gomock.Any(), gomock.Any(), gomock.Any()).Times(0) repoMock.EXPECT().DecrTaskRunCount(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Times(0) - proc := &AutoEvaluteProcessor{ + proc := &AutoEvaluateProcessor{ evaluationSvc: eval, taskRepo: repoAdapter, } @@ -426,7 +438,7 @@ func TestAutoEvaluteProcessor_Invoke(t *testing.T) { }) } -func TestAutoEvaluteProcessor_OnUpdateTaskChange(t *testing.T) { +func TestAutoEvaluateProcessor_OnUpdateTaskChange(t *testing.T) { t.Parallel() ctx := context.Background() @@ -456,7 +468,7 @@ func TestAutoEvaluteProcessor_OnUpdateTaskChange(t *testing.T) { return nil }) - proc := &AutoEvaluteProcessor{taskRepo: repoAdapter} + proc := &AutoEvaluateProcessor{taskRepo: repoAdapter} taskObj := &taskentity.ObservabilityTask{TaskStatus: caseItem.initial} err := proc.OnTaskUpdated(ctx, taskObj, caseItem.op) assert.NoError(t, err) @@ -464,13 +476,13 @@ func TestAutoEvaluteProcessor_OnUpdateTaskChange(t *testing.T) { } t.Run("invalid op", func(t *testing.T) { - proc := &AutoEvaluteProcessor{} + proc := &AutoEvaluateProcessor{} err := proc.OnTaskUpdated(ctx, &taskentity.ObservabilityTask{}, "unknown") assert.Error(t, err) }) } -func TestAutoEvaluteProcessor_OnCreateTaskRunChange(t *testing.T) { +func TestAutoEvaluateProcessor_OnCreateTaskRunChange(t *testing.T) { t.Parallel() ctrl := gomock.NewController(t) defer ctrl.Finish() @@ -499,7 +511,7 @@ func TestAutoEvaluteProcessor_OnCreateTaskRunChange(t *testing.T) { evalAdapter.submitResp.exptID = 1111 evalAdapter.submitResp.exptRunID = 2222 - proc := &AutoEvaluteProcessor{ + proc := &AutoEvaluateProcessor{ datasetServiceAdaptor: adaptor, evaluationSvc: evalAdapter, taskRepo: repoAdapter, @@ -513,7 +525,7 @@ func TestAutoEvaluteProcessor_OnCreateTaskRunChange(t *testing.T) { assert.Equal(t, int64(9001), *evalAdapter.submitReq.EvalSetID) } -func TestAutoEvaluteProcessor_OnFinishTaskRunChange(t *testing.T) { +func TestAutoEvaluateProcessor_OnFinishTaskRunChange(t *testing.T) { t.Parallel() ctrl := gomock.NewController(t) defer ctrl.Finish() @@ -533,7 +545,7 @@ func TestAutoEvaluteProcessor_OnFinishTaskRunChange(t *testing.T) { } repoMock.EXPECT().UpdateTaskRun(gomock.Any(), taskRun).Return(nil) - proc := &AutoEvaluteProcessor{ + proc := &AutoEvaluateProcessor{ taskRepo: repoAdapter, evaluationSvc: evalAdapter, } @@ -547,7 +559,7 @@ func TestAutoEvaluteProcessor_OnFinishTaskRunChange(t *testing.T) { assert.Equal(t, taskentity.TaskRunStatusDone, taskRun.RunStatus) } -func TestAutoEvaluteProcessor_OnFinishTaskChange(t *testing.T) { +func TestAutoEvaluateProcessor_OnFinishTaskChange(t *testing.T) { t.Parallel() ctrl := gomock.NewController(t) defer ctrl.Finish() @@ -562,7 +574,7 @@ func TestAutoEvaluteProcessor_OnFinishTaskChange(t *testing.T) { repoMock.EXPECT().UpdateTaskRun(gomock.Any(), gomock.Any()).Return(nil) repoMock.EXPECT().UpdateTask(gomock.Any(), taskObj).Return(nil) - proc := &AutoEvaluteProcessor{ + proc := &AutoEvaluateProcessor{ evaluationSvc: evalAdapter, taskRepo: repoAdapter, } @@ -576,7 +588,7 @@ func TestAutoEvaluteProcessor_OnFinishTaskChange(t *testing.T) { assert.Equal(t, taskentity.TaskStatusSuccess, taskObj.TaskStatus) } -func TestAutoEvaluteProcessor_OnFinishTaskChange_Error(t *testing.T) { +func TestAutoEvaluateProcessor_OnFinishTaskChange_Error(t *testing.T) { t.Parallel() ctrl := gomock.NewController(t) defer ctrl.Finish() @@ -586,7 +598,7 @@ func TestAutoEvaluteProcessor_OnFinishTaskChange_Error(t *testing.T) { evalAdapter := &fakeEvaluationAdapter{} evalAdapter.finishErr = errors.New("finish fail") - proc := &AutoEvaluteProcessor{ + proc := &AutoEvaluateProcessor{ evaluationSvc: evalAdapter, taskRepo: repoAdapter, } @@ -598,7 +610,7 @@ func TestAutoEvaluteProcessor_OnFinishTaskChange_Error(t *testing.T) { assert.EqualError(t, err, "finish fail") } -func TestAutoEvaluteProcessor_OnCreateTaskChange(t *testing.T) { +func TestAutoEvaluateProcessor_OnCreateTaskChange(t *testing.T) { t.Parallel() ctrl := gomock.NewController(t) defer ctrl.Finish() @@ -614,7 +626,7 @@ func TestAutoEvaluteProcessor_OnCreateTaskChange(t *testing.T) { evalAdapter.submitResp.exptID = 111 evalAdapter.submitResp.exptRunID = 222 - proc := &AutoEvaluteProcessor{ + proc := &AutoEvaluateProcessor{ datasetServiceAdaptor: adaptor, evaluationSvc: evalAdapter, taskRepo: repoAdapter, @@ -674,7 +686,7 @@ func TestAutoEvaluteProcessor_OnCreateTaskChange(t *testing.T) { assert.Equal(t, taskentity.TaskStatusRunning, taskObj.TaskStatus) } -func TestAutoEvaluteProcessor_OnCreateTaskChange_GetBackfillError(t *testing.T) { +func TestAutoEvaluateProcessor_OnCreateTaskChange_GetBackfillError(t *testing.T) { t.Parallel() ctrl := gomock.NewController(t) defer ctrl.Finish() @@ -684,13 +696,13 @@ func TestAutoEvaluteProcessor_OnCreateTaskChange_GetBackfillError(t *testing.T) repoMock.EXPECT().GetBackfillTaskRun(gomock.Any(), (*int64)(nil), gomock.Any()).Return(nil, errors.New("db error")) - proc := &AutoEvaluteProcessor{taskRepo: repoAdapter} + proc := &AutoEvaluateProcessor{taskRepo: repoAdapter} err := proc.OnTaskCreated(context.Background(), buildTestTask(t)) assert.EqualError(t, err, "db error") } -func TestAutoEvaluteProcessor_OnCreateTaskChange_CreateDatasetError(t *testing.T) { +func TestAutoEvaluateProcessor_OnCreateTaskChange_CreateDatasetError(t *testing.T) { t.Parallel() ctrl := gomock.NewController(t) defer ctrl.Finish() @@ -702,7 +714,7 @@ func TestAutoEvaluteProcessor_OnCreateTaskChange_CreateDatasetError(t *testing.T adaptor := service.NewDatasetServiceAdaptor() adaptor.Register(traceentity.DatasetCategory_Evaluation, datasetProvider) - proc := &AutoEvaluteProcessor{ + proc := &AutoEvaluateProcessor{ datasetServiceAdaptor: adaptor, taskRepo: repoAdapter, evaluationSvc: &fakeEvaluationAdapter{}, @@ -715,9 +727,9 @@ func TestAutoEvaluteProcessor_OnCreateTaskChange_CreateDatasetError(t *testing.T assert.EqualError(t, err, "create fail") } -func TestAutoEvaluteProcessor_getSession(t *testing.T) { +func TestAutoEvaluateProcessor_getSession(t *testing.T) { t.Parallel() - proc := &AutoEvaluteProcessor{aid: 567} + proc := &AutoEvaluateProcessor{aid: 567} taskObj := &taskentity.ObservabilityTask{CreatedBy: "42"} @@ -729,3 +741,68 @@ func TestAutoEvaluteProcessor_getSession(t *testing.T) { s = proc.getSession(context.Background(), taskObj) assert.EqualValues(t, 42, *s.UserID) } + +func TestAutoEvaluateProcessor_OnTaskUpdated_InvalidStatus(t *testing.T) { + t.Parallel() + ctx := context.Background() + + proc := &AutoEvaluateProcessor{} + + taskObj := &taskentity.ObservabilityTask{TaskStatus: taskentity.TaskStatusRunning} + + err := proc.OnTaskUpdated(ctx, taskObj, "invalid_status") + assert.Error(t, err) +} + +func TestAutoEvaluateProcessor_OnTaskFinished_NoAutoEvalConfig(t *testing.T) { + t.Parallel() + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + repoMock := repomocks.NewMockITaskRepo(ctrl) + repoAdapter := &taskRepoMockAdapter{MockITaskRepo: repoMock} + evalAdapter := &fakeEvaluationAdapter{} + + proc := &AutoEvaluateProcessor{ + evaluationSvc: evalAdapter, + taskRepo: repoAdapter, + } + + taskObj := &taskentity.ObservabilityTask{TaskStatus: taskentity.TaskStatusRunning, WorkspaceID: 123} + taskRun := &taskentity.TaskRun{TaskRunConfig: nil} // No auto eval config + + // Mock the UpdateTask call + repoMock.EXPECT().UpdateTask(gomock.Any(), taskObj).Return(nil) + + err := proc.OnTaskFinished(context.Background(), taskexe.OnTaskFinishedReq{ + Task: taskObj, + TaskRun: taskRun, + IsFinish: true, + }) + assert.NoError(t, err) + assert.Equal(t, taskentity.TaskStatusSuccess, taskObj.TaskStatus) +} + +func TestAutoEvaluateProcessor_NewAutoEvaluateProcessor(t *testing.T) { + t.Parallel() + + // Create mock dependencies + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + datasetServiceAdaptor := service.NewDatasetServiceAdaptor() + evalService := &fakeEvaluatorAdapter{} + evaluationService := &fakeEvaluationAdapter{} + taskRepo := repomocks.NewMockITaskRepo(ctrl) + + // Test constructor + proc := NewAutoEvaluateProcessor(123, datasetServiceAdaptor, evalService, evaluationService, taskRepo, &EvalTargetBuilderImpl{}) + + assert.NotNil(t, proc) + assert.Equal(t, int32(123), proc.aid) + assert.Equal(t, datasetServiceAdaptor, proc.datasetServiceAdaptor) + assert.Equal(t, evalService, proc.evalSvc) + assert.Equal(t, evaluationService, proc.evaluationSvc) + assert.Equal(t, taskRepo, proc.taskRepo) + +} diff --git a/backend/modules/observability/domain/task/service/taskexe/processor/eval_target_builder.go b/backend/modules/observability/domain/task/service/taskexe/processor/eval_target_builder.go new file mode 100644 index 000000000..6a38d574c --- /dev/null +++ b/backend/modules/observability/domain/task/service/taskexe/processor/eval_target_builder.go @@ -0,0 +1,29 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 + +package processor + +import ( + "context" + "strconv" + + eval_target_d "github.com/coze-dev/coze-loop/backend/kitex_gen/coze/loop/evaluation/domain/eval_target" + "github.com/coze-dev/coze-loop/backend/kitex_gen/coze/loop/evaluation/eval_target" + task_entity "github.com/coze-dev/coze-loop/backend/modules/observability/domain/task/entity" + "github.com/samber/lo" +) + +type EvalTargetBuilder interface { + Build(ctx context.Context, task *task_entity.ObservabilityTask) *eval_target.CreateEvalTargetParam +} + +type EvalTargetBuilderImpl struct { + EvalTargetBuilder +} + +func (b *EvalTargetBuilderImpl) Build(ctx context.Context, task *task_entity.ObservabilityTask) *eval_target.CreateEvalTargetParam { + return &eval_target.CreateEvalTargetParam{ + EvalTargetType: lo.ToPtr(eval_target_d.EvalTargetType_Trace), + SourceTargetID: lo.ToPtr(strconv.FormatInt(task.ID, 10)), + } +} diff --git a/backend/modules/observability/domain/task/service/taskexe/tracehub/backfill.go b/backend/modules/observability/domain/task/service/taskexe/tracehub/backfill.go index 1718a3d4c..5240c7c45 100644 --- a/backend/modules/observability/domain/task/service/taskexe/tracehub/backfill.go +++ b/backend/modules/observability/domain/task/service/taskexe/tracehub/backfill.go @@ -7,6 +7,7 @@ import ( "context" "errors" "fmt" + "strconv" "time" "github.com/coze-dev/coze-loop/backend/infra/middleware/session" @@ -144,6 +145,7 @@ func (h *TraceHubServiceImpl) listAndSendSpans(ctx context.Context, sub *spanSub // Build query parameters listParam := &repo.ListSpansParam{ + WorkSpaceID: strconv.FormatInt(sub.t.WorkspaceID, 10), Tenants: tenants, Filters: h.buildSpanFilters(ctx, sub.t), StartAt: backfillTime.StartAt, diff --git a/backend/modules/observability/domain/task/service/taskexe/tracehub/mocks/trace_hub_service.go b/backend/modules/observability/domain/task/service/taskexe/tracehub/mocks/trace_hub_service.go index 2e8ac29ae..e74dab70f 100644 --- a/backend/modules/observability/domain/task/service/taskexe/tracehub/mocks/trace_hub_service.go +++ b/backend/modules/observability/domain/task/service/taskexe/tracehub/mocks/trace_hub_service.go @@ -83,4 +83,4 @@ func (m *MockITraceHubService) StoneTaskCache(ctx context.Context, cacheInfo tra func (mr *MockITraceHubServiceMockRecorder) StoneTaskCache(ctx, span any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StoneTaskCache", reflect.TypeOf((*MockITraceHubService)(nil).StoneTaskCache), ctx, span) -} \ No newline at end of file +} diff --git a/backend/modules/observability/domain/task/service/taskexe/tracehub/trace_hub.go b/backend/modules/observability/domain/task/service/taskexe/tracehub/trace_hub.go index 8b9863543..4ad2e5017 100644 --- a/backend/modules/observability/domain/task/service/taskexe/tracehub/trace_hub.go +++ b/backend/modules/observability/domain/task/service/taskexe/tracehub/trace_hub.go @@ -49,7 +49,6 @@ func NewTraceHubImpl( config: config, localCache: NewLocalCache(), } - return impl, nil } diff --git a/backend/modules/observability/domain/trace/entity/loop_span/filter.go b/backend/modules/observability/domain/trace/entity/loop_span/filter.go index 5e83ba6b0..8d74abdf3 100644 --- a/backend/modules/observability/domain/trace/entity/loop_span/filter.go +++ b/backend/modules/observability/domain/trace/entity/loop_span/filter.go @@ -47,12 +47,16 @@ const ( FieldTypeDouble FieldType = "double" FieldTypeBool FieldType = "bool" - PlatformDefault PlatformType = "default" - PlatformCozeLoop PlatformType = "cozeloop" - PlatformPrompt PlatformType = "prompt" - PlatformEvaluator PlatformType = "evaluator" - PlatformEvalTarget PlatformType = "evaluation_target" - PlatformOpenAPI PlatformType = "open_api" + PlatformDefault PlatformType = "default" + PlatformCozeLoop PlatformType = "cozeloop" + PlatformPrompt PlatformType = "prompt" + PlatformEvaluator PlatformType = "evaluator" + PlatformEvalTarget PlatformType = "evaluation_target" + PlatformOpenAPI PlatformType = "open_api" + PlatformCozeWorkflow PlatformType = "coze_workflow" + PlatformCozeBot PlatformType = "coze_bot" + PlatformVeAgentKit PlatformType = "ve_agentkit" + PlatformVeADK PlatformType = "veadk" SpanListTypeRootSpan SpanListType = "root_span" SpanListTypeAllSpan SpanListType = "all_span" @@ -152,10 +156,6 @@ func (f *FilterFields) Traverse(fn func(f *FilterField) error) error { return nil } -//func (f *FilterFields) Filter[T FilterObject](objs []T) []T { -// -//} - func (f *FilterFields) Satisfied(obj FilterObject) bool { op := QueryAndOrEnumAnd hit := true diff --git a/backend/modules/observability/domain/trace/entity/otel/otel_span.go b/backend/modules/observability/domain/trace/entity/otel/otel_span.go deleted file mode 100644 index 5c467f1f3..000000000 --- a/backend/modules/observability/domain/trace/entity/otel/otel_span.go +++ /dev/null @@ -1,10 +0,0 @@ -// Copyright (c) 2025 coze-dev Authors -// SPDX-License-Identifier: Apache-2.0 - -package otel - -type ResourceScopeSpan struct { - Resource *Resource `json:"resource,omitempty"` - Scope *InstrumentationScope `json:"scope,omitempty"` - Span *Span `json:"span,omitempty"` -} diff --git a/backend/modules/observability/domain/trace/repo/trace.go b/backend/modules/observability/domain/trace/repo/trace.go index 70dff260b..ec2a320e0 100644 --- a/backend/modules/observability/domain/trace/repo/trace.go +++ b/backend/modules/observability/domain/trace/repo/trace.go @@ -10,6 +10,7 @@ import ( ) type GetTraceParam struct { + WorkSpaceID string Tenants []string TraceID string LogID string @@ -24,6 +25,7 @@ type GetTraceParam struct { } type ListSpansParam struct { + WorkSpaceID string Tenants []string Filters *loop_span.FilterFields StartAt int64 // ms @@ -46,19 +48,22 @@ type GetPreSpanIDsParam struct { PreRespID string } type InsertTraceParam struct { - Spans loop_span.SpanList - Tenant string - TTL loop_span.TTL + WorkSpaceID string + Spans loop_span.SpanList + Tenant string + TTL loop_span.TTL } type GetAnnotationParam struct { - Tenants []string - ID string - StartAt int64 // ms - EndAt int64 // ms + WorkSpaceID string + Tenants []string + ID string + StartAt int64 // ms + EndAt int64 // ms } type ListAnnotationsParam struct { + WorkSpaceID string Tenants []string SpanID string TraceID string @@ -69,6 +74,7 @@ type ListAnnotationsParam struct { } type InsertAnnotationParam struct { + WorkSpaceID string Tenant string TTL loop_span.TTL Span *loop_span.Span diff --git a/backend/modules/observability/domain/trace/service/trace_export_service.go b/backend/modules/observability/domain/trace/service/trace_export_service.go index a5dc99149..512e1eb8b 100644 --- a/backend/modules/observability/domain/trace/service/trace_export_service.go +++ b/backend/modules/observability/domain/trace/service/trace_export_service.go @@ -5,6 +5,8 @@ package service import ( "context" + "github.com/coze-dev/coze-loop/backend/modules/observability/domain/component/storage" + "strconv" "github.com/bytedance/gg/gptr" "github.com/coze-dev/coze-loop/backend/infra/middleware/session" @@ -86,6 +88,7 @@ type ITraceExportService interface { func NewTraceExportServiceImpl( tRepo repo.ITraceRepo, + storageProvider storage.IStorageProvider, traceConfig config.ITraceConfig, traceProducer mq.ITraceProducer, annotationProducer mq.IAnnotationProducer, @@ -261,7 +264,8 @@ func (r *TraceExportServiceImpl) getSpans(ctx context.Context, workspaceID int64 spanIDs := lo.Map(sids, func(s SpanID, _ int) string { return s.SpanID }) traceIDs := lo.UniqMap(sids, func(s SpanID, _ int) string { return s.TraceID }) result, err := r.traceRepo.ListSpans(ctx, &repo.ListSpansParam{ - Tenants: tenant, + WorkSpaceID: strconv.FormatInt(workspaceID, 10), + Tenants: tenant, Filters: &loop_span.FilterFields{ FilterFields: []*loop_span.FilterField{ { @@ -402,6 +406,7 @@ func (r *TraceExportServiceImpl) addSpanAnnotations(ctx context.Context, spans [ continue } err = r.traceRepo.InsertAnnotations(ctx, &repo.InsertAnnotationParam{ + WorkSpaceID: span.WorkspaceID, Tenant: span.GetTenant(), TTL: span.GetTTL(ctx), Span: span, diff --git a/backend/modules/observability/domain/trace/service/trace_service.go b/backend/modules/observability/domain/trace/service/trace_service.go index 463357ca5..15997effb 100644 --- a/backend/modules/observability/domain/trace/service/trace_service.go +++ b/backend/modules/observability/domain/trace/service/trace_service.go @@ -528,6 +528,7 @@ func (r *TraceServiceImpl) GetTrace(ctx context.Context, req *GetTraceReq) (*Get limit = 10000 } spans, err := r.traceRepo.GetTrace(ctx, &repo.GetTraceParam{ + WorkSpaceID: strconv.FormatInt(req.WorkspaceID, 10), Tenants: tenants, LogID: req.LogID, TraceID: req.TraceID, @@ -589,6 +590,7 @@ func (r *TraceServiceImpl) ListSpans(ctx context.Context, req *ListSpansReq) (*L } st := time.Now() tRes, err := r.traceRepo.ListSpans(ctx, &repo.ListSpansParam{ + WorkSpaceID: strconv.FormatInt(req.WorkspaceID, 10), Tenants: tenants, Filters: filters, StartAt: req.StartTime, @@ -638,6 +640,7 @@ func (r *TraceServiceImpl) SearchTraceOApi(ctx context.Context, req *SearchTrace } spans, err := r.traceRepo.GetTrace(ctx, &repo.GetTraceParam{ + WorkSpaceID: strconv.FormatInt(req.WorkspaceID, 10), Tenants: req.Tenants, TraceID: req.TraceID, LogID: req.LogID, @@ -658,7 +661,7 @@ func (r *TraceServiceImpl) SearchTraceOApi(ctx context.Context, req *SearchTrace QueryStartTime: req.StartTime, QueryEndTime: req.EndTime, PlatformType: req.PlatformType, - SpanDoubleCheck: req.Filters != nil && len(req.Filters.FilterFields) > 0, + SpanDoubleCheck: len(req.SpanIDs) > 0 || (req.Filters != nil && len(req.Filters.FilterFields) > 0), QueryTenants: req.Tenants, QueryTraceID: req.TraceID, QueryLogID: req.LogID, @@ -698,6 +701,7 @@ func (r *TraceServiceImpl) ListSpansOApi(ctx context.Context, req *ListSpansOApi } filters := r.combineFilters(builtinFilter, req.Filters) tRes, err := r.traceRepo.ListSpans(ctx, &repo.ListSpansParam{ + WorkSpaceID: strconv.FormatInt(req.WorkspaceID, 10), Tenants: req.Tenants, Filters: filters, StartAt: req.StartTime, @@ -780,6 +784,7 @@ func (r *TraceServiceImpl) GetTracesAdvanceInfo(ctx context.Context, req *GetTra g.Go(func() error { defer goroutine.Recovery(ctx) qReq := &repo.GetTraceParam{ + WorkSpaceID: strconv.FormatInt(req.WorkspaceID, 10), Tenants: tenants, TraceID: v.TraceID, StartAt: v.StartTime, @@ -890,6 +895,7 @@ func (r *TraceServiceImpl) ListAnnotations(ctx context.Context, req *ListAnnotat return nil, err } annotations, err := r.traceRepo.ListAnnotations(ctx, &repo.ListAnnotationsParam{ + WorkSpaceID: strconv.FormatInt(req.WorkspaceID, 10), Tenants: tenants, SpanID: req.SpanID, TraceID: req.TraceID, @@ -938,6 +944,7 @@ func (r *TraceServiceImpl) CreateManualAnnotation(ctx context.Context, req *Crea return nil, errorx.WrapByCode(err, obErrorx.CommercialCommonInvalidParamCodeCode, errorx.WithExtraMsg("invalid annotation")) } if err := r.traceRepo.InsertAnnotations(ctx, &repo.InsertAnnotationParam{ + WorkSpaceID: span.WorkspaceID, Tenant: span.GetTenant(), TTL: span.GetTTL(ctx), Span: span, @@ -982,10 +989,11 @@ func (r *TraceServiceImpl) UpdateManualAnnotation(ctx context.Context, req *Upda return errorx.NewByCode(obErrorx.CommercialCommonInvalidParamCodeCode) } existedAnno, err := r.traceRepo.GetAnnotation(ctx, &repo.GetAnnotationParam{ - Tenants: tenants, - ID: req.AnnotationID, - StartAt: time.UnixMicro(span.StartTime).Add(-time.Second).UnixMilli(), - EndAt: time.UnixMicro(span.StartTime).Add(time.Second).UnixMilli(), + WorkSpaceID: req.Annotation.WorkspaceID, + Tenants: tenants, + ID: req.AnnotationID, + StartAt: time.UnixMicro(span.StartTime).Add(-time.Second).UnixMilli(), + EndAt: time.UnixMicro(span.StartTime).Add(time.Second).UnixMilli(), }) if err != nil { logs.CtxError(ctx, "get annotation %s err %v", req.AnnotationID, err) @@ -995,6 +1003,7 @@ func (r *TraceServiceImpl) UpdateManualAnnotation(ctx context.Context, req *Upda annotation.CreatedAt = existedAnno.CreatedAt } return r.traceRepo.InsertAnnotations(ctx, &repo.InsertAnnotationParam{ + WorkSpaceID: span.WorkspaceID, Tenant: span.GetTenant(), TTL: span.GetTTL(ctx), Span: span, @@ -1034,6 +1043,7 @@ func (r *TraceServiceImpl) DeleteManualAnnotation(ctx context.Context, req *Dele return errorx.NewByCode(obErrorx.CommercialCommonInvalidParamCodeCode, errorx.WithExtraMsg("invalid annotation")) } return r.traceRepo.InsertAnnotations(ctx, &repo.InsertAnnotationParam{ + WorkSpaceID: span.WorkspaceID, Tenant: span.GetTenant(), TTL: span.GetTTL(ctx), Span: span, @@ -1087,10 +1097,11 @@ func (r *TraceServiceImpl) CreateAnnotation(ctx context.Context, req *CreateAnno return errorx.WrapByCode(err, obErrorx.CommercialCommonInvalidParamCodeCode, errorx.WithExtraMsg("invalid annotation")) } existedAnno, err := r.traceRepo.GetAnnotation(ctx, &repo.GetAnnotationParam{ - Tenants: cfg.Tenants, - ID: annotation.ID, - StartAt: time.UnixMicro(span.StartTime).Add(-time.Second).UnixMilli(), - EndAt: time.UnixMicro(span.StartTime).Add(time.Second).UnixMilli(), + WorkSpaceID: strconv.FormatInt(req.WorkspaceID, 10), + Tenants: cfg.Tenants, + ID: annotation.ID, + StartAt: time.UnixMicro(span.StartTime).Add(-time.Second).UnixMilli(), + EndAt: time.UnixMicro(span.StartTime).Add(time.Second).UnixMilli(), }) if err != nil { return err @@ -1099,6 +1110,7 @@ func (r *TraceServiceImpl) CreateAnnotation(ctx context.Context, req *CreateAnno annotation.CreatedAt = existedAnno.CreatedAt } return r.traceRepo.InsertAnnotations(ctx, &repo.InsertAnnotationParam{ + WorkSpaceID: span.WorkspaceID, Tenant: span.GetTenant(), TTL: span.GetTTL(ctx), Span: span, @@ -1151,6 +1163,7 @@ func (r *TraceServiceImpl) DeleteAnnotation(ctx context.Context, req *DeleteAnno return errorx.WrapByCode(err, obErrorx.CommercialCommonInvalidParamCodeCode, errorx.WithExtraMsg("invalid annotation")) } return r.traceRepo.InsertAnnotations(ctx, &repo.InsertAnnotationParam{ + WorkSpaceID: span.WorkspaceID, Tenant: span.GetTenant(), TTL: span.GetTTL(ctx), Span: span, @@ -1193,6 +1206,7 @@ func (r *TraceServiceImpl) Send(ctx context.Context, event *entity.AnnotationEve } // retry if failed return r.traceRepo.InsertAnnotations(ctx, &repo.InsertAnnotationParam{ + WorkSpaceID: span.WorkspaceID, Tenant: span.GetTenant(), TTL: span.GetTTL(ctx), Span: span, @@ -1245,7 +1259,8 @@ func (r *TraceServiceImpl) getSpan(ctx context.Context, tenants []string, spanId }) } res, err := r.traceRepo.ListSpans(ctx, &repo.ListSpansParam{ - Tenants: tenants, + WorkSpaceID: workspaceId, + Tenants: tenants, Filters: &loop_span.FilterFields{ FilterFields: filterFields, }, @@ -1363,10 +1378,11 @@ func (r *TraceServiceImpl) ChangeEvaluatorScore(ctx context.Context, req *Change } span := spans[0] annotation, err := r.traceRepo.GetAnnotation(ctx, &repo.GetAnnotationParam{ - Tenants: tenants, - ID: req.AnnotationID, - StartAt: time.UnixMicro(span.StartTime).Add(-time.Second).UnixMilli(), - EndAt: time.UnixMicro(span.StartTime).Add(time.Second).UnixMilli(), + WorkSpaceID: strconv.FormatInt(req.WorkspaceID, 10), + Tenants: tenants, + ID: req.AnnotationID, + StartAt: time.UnixMicro(span.StartTime).Add(-time.Second).UnixMilli(), + EndAt: time.UnixMicro(span.StartTime).Add(time.Second).UnixMilli(), }) if err != nil { logs.CtxError(ctx, "get annotation %s err %v", req.AnnotationID, err) @@ -1387,6 +1403,7 @@ func (r *TraceServiceImpl) ChangeEvaluatorScore(ctx context.Context, req *Change // 再同步修改观测数据 span.Annotations = append(span.Annotations, annotation) param := &repo.InsertAnnotationParam{ + WorkSpaceID: span.WorkspaceID, Tenant: span.GetTenant(), TTL: span.GetTTL(ctx), Span: span, diff --git a/backend/modules/observability/domain/trace/service/trace_service_test.go b/backend/modules/observability/domain/trace/service/trace_service_test.go index f89f4e945..9be1c663c 100644 --- a/backend/modules/observability/domain/trace/service/trace_service_test.go +++ b/backend/modules/observability/domain/trace/service/trace_service_test.go @@ -2758,6 +2758,7 @@ func TestTraceServiceImpl_SearchTraceOApi(t *testing.T) { fieldsGetter: func(ctrl *gomock.Controller) fields { repoMock := repomocks.NewMockITraceRepo(ctrl) repoMock.EXPECT().GetTrace(gomock.Any(), &repo.GetTraceParam{ + WorkSpaceID: "123", Tenants: []string{"tenant1"}, TraceID: "trace-123", LogID: "", diff --git a/backend/modules/observability/infra/repo/ck/annotation.go b/backend/modules/observability/infra/repo/ck/annotation.go index 38365907d..d2018e997 100644 --- a/backend/modules/observability/infra/repo/ck/annotation.go +++ b/backend/modules/observability/infra/repo/ck/annotation.go @@ -7,39 +7,10 @@ import ( "context" "github.com/coze-dev/coze-loop/backend/infra/ck" - "github.com/coze-dev/coze-loop/backend/modules/observability/infra/repo/ck/gorm_gen/model" + "github.com/coze-dev/coze-loop/backend/modules/observability/infra/repo/dao" ) -type InsertAnnotationParam struct { - Table string - Annotations []*model.ObservabilityAnnotation -} - -type GetAnnotationParam struct { - Tables []string - ID string - StartTime int64 // us - EndTime int64 // us - Limit int32 -} - -type ListAnnotationsParam struct { - Tables []string - SpanIDs []string - StartTime int64 // us - EndTime int64 // us - DescByUpdatedAt bool - Limit int32 -} - -//go:generate mockgen -destination=mocks/annotation_dao.go -package=mocks . IAnnotationDao -type IAnnotationDao interface { - Insert(context.Context, *InsertAnnotationParam) error - Get(context.Context, *GetAnnotationParam) (*model.ObservabilityAnnotation, error) - List(context.Context, *ListAnnotationsParam) ([]*model.ObservabilityAnnotation, error) -} - -func NewAnnotationCkDaoImpl(db ck.Provider) (IAnnotationDao, error) { +func NewAnnotationCkDaoImpl(db ck.Provider) (dao.IAnnotationDao, error) { return &AnnotationCkDaoImpl{ db: db, }, nil @@ -49,14 +20,14 @@ type AnnotationCkDaoImpl struct { db ck.Provider } -func (a *AnnotationCkDaoImpl) Insert(ctx context.Context, params *InsertAnnotationParam) error { +func (a *AnnotationCkDaoImpl) Insert(ctx context.Context, params *dao.InsertAnnotationParam) error { return nil } -func (a *AnnotationCkDaoImpl) Get(ctx context.Context, params *GetAnnotationParam) (*model.ObservabilityAnnotation, error) { +func (a *AnnotationCkDaoImpl) Get(ctx context.Context, params *dao.GetAnnotationParam) (*dao.Annotation, error) { return nil, nil } -func (a *AnnotationCkDaoImpl) List(ctx context.Context, params *ListAnnotationsParam) ([]*model.ObservabilityAnnotation, error) { +func (a *AnnotationCkDaoImpl) List(ctx context.Context, params *dao.ListAnnotationsParam) ([]*dao.Annotation, error) { return nil, nil } diff --git a/backend/modules/observability/infra/repo/ck/convertor/span.go b/backend/modules/observability/infra/repo/ck/convertor/span.go index 1d7a51bce..734b43594 100644 --- a/backend/modules/observability/infra/repo/ck/convertor/span.go +++ b/backend/modules/observability/infra/repo/ck/convertor/span.go @@ -4,143 +4,90 @@ package convertor import ( - "time" - - "github.com/coze-dev/coze-loop/backend/modules/observability/domain/trace/entity/loop_span" "github.com/coze-dev/coze-loop/backend/modules/observability/infra/repo/ck/gorm_gen/model" - "github.com/coze-dev/coze-loop/backend/pkg/lang/ptr" + "github.com/coze-dev/coze-loop/backend/modules/observability/infra/repo/dao" ) -func SpanListDO2PO(spans loop_span.SpanList, TTL loop_span.TTL) []*model.ObservabilitySpan { +func SpanListPO2CKModels(spans []*dao.Span) []*model.ObservabilitySpan { ret := make([]*model.ObservabilitySpan, len(spans)) for i, span := range spans { - ret[i] = SpanDO2PO(span, TTL) + ret[i] = SpanPO2CKModel(span) } return ret } -func SpanListPO2DO(spans []*model.ObservabilitySpan) loop_span.SpanList { - ret := make(loop_span.SpanList, len(spans)) +func SpanListCKModels2PO(spans []*model.ObservabilitySpan) []*dao.Span { + ret := make([]*dao.Span, len(spans)) for i, span := range spans { - ret[i] = SpanPO2DO(span) + ret[i] = SpanCKModel2PO(span) } return ret } -func SpanDO2PO(span *loop_span.Span, TTL loop_span.TTL) *model.ObservabilitySpan { - ret := &model.ObservabilitySpan{ - TraceID: span.TraceID, - SpanID: span.SpanID, - SpaceID: span.WorkspaceID, - SpanType: span.SpanType, - SpanName: span.SpanName, - ParentID: span.ParentID, - StartTime: span.StartTime, // us - Duration: span.DurationMicros, - Psm: ptr.Of(span.PSM), - Logid: ptr.Of(span.LogID), - StatusCode: span.StatusCode, - Input: span.Input, - Output: span.Output, - TagsFloat: CopyMap(span.TagsDouble), - TagsString: CopyMap(span.TagsString), - TagsLong: CopyMap(span.TagsLong), - TagsByte: CopyMap(span.TagsByte), - SystemTagsFloat: CopyMap(span.SystemTagsDouble), - SystemTagsLong: CopyMap(span.SystemTagsLong), - SystemTagsString: CopyMap(span.SystemTagsString), - } - ret.TagsBool = make(map[string]uint8) - for k, v := range span.TagsBool { - if v { - ret.TagsBool[k] = 1 - } else { - ret.TagsBool[k] = 0 - } - } - if span.Method != "" { - ret.Method = ptr.Of(span.Method) - } - if span.CallType != "" { - ret.CallType = ptr.Of(span.CallType) - } - if span.ObjectStorage != "" { - ret.ObjectStorage = ptr.Of(span.ObjectStorage) +func SpanPO2CKModel(span *dao.Span) *model.ObservabilitySpan { + if span == nil { + return nil } - switch TTL { - case loop_span.TTL3d: - ret.LogicDeleteDate = time.Now().Add(3 * 24 * time.Hour).UnixMicro() - case loop_span.TTL7d: - ret.LogicDeleteDate = time.Now().Add(7 * 24 * time.Hour).UnixMicro() - case loop_span.TTL30d: - ret.LogicDeleteDate = time.Now().Add(30 * 24 * time.Hour).UnixMicro() - case loop_span.TTL90d: - ret.LogicDeleteDate = time.Now().Add(90 * 24 * time.Hour).UnixMicro() - case loop_span.TTL180d: - ret.LogicDeleteDate = time.Now().Add(180 * 24 * time.Hour).UnixMicro() - case loop_span.TTL365d: - ret.LogicDeleteDate = time.Now().Add(365 * 24 * time.Hour).UnixMicro() - default: - ret.LogicDeleteDate = time.Now().Add(3 * 24 * time.Hour).UnixMicro() + return &model.ObservabilitySpan{ + TraceID: span.TraceID, + SpanID: span.SpanID, + SpaceID: span.SpaceID, + SpanType: span.SpanType, + SpanName: span.SpanName, + ParentID: span.ParentID, + Method: span.Method, + Psm: span.Psm, + Logid: span.Logid, + StartTime: span.StartTime, // us + CallType: span.CallType, + Duration: span.Duration, + StatusCode: span.StatusCode, + ObjectStorage: span.ObjectStorage, + Input: span.Input, + Output: span.Output, + LogicDeleteDate: span.LogicDeleteDate, + ReserveCreateTime: span.ReserveCreateTime, + TagsBool: span.TagsBool, + TagsFloat: span.TagsFloat, + TagsString: span.TagsString, + TagsLong: span.TagsLong, + TagsByte: span.TagsByte, + SystemTagsFloat: span.SystemTagsFloat, + SystemTagsLong: span.SystemTagsLong, + SystemTagsString: span.SystemTagsString, } - return ret } -func SpanPO2DO(span *model.ObservabilitySpan) *loop_span.Span { +func SpanCKModel2PO(span *model.ObservabilitySpan) *dao.Span { if span == nil { return nil } - ret := &loop_span.Span{ - TraceID: span.TraceID, - SpanID: span.SpanID, - WorkspaceID: span.SpaceID, - SpanType: span.SpanType, - SpanName: span.SpanName, - ParentID: span.ParentID, - StartTime: span.StartTime, // us - DurationMicros: span.Duration, - StatusCode: span.StatusCode, - Input: span.Input, - Output: span.Output, - TagsDouble: CopyMap(span.TagsFloat), - TagsString: CopyMap(span.TagsString), - TagsLong: CopyMap(span.TagsLong), - TagsByte: CopyMap(span.TagsByte), - SystemTagsDouble: CopyMap(span.SystemTagsFloat), - SystemTagsLong: CopyMap(span.SystemTagsLong), - SystemTagsString: CopyMap(span.SystemTagsString), - LogicDeleteTime: span.LogicDeleteDate, - } - ret.TagsBool = make(map[string]bool) - for k, v := range span.TagsBool { - if v > 0 { - ret.TagsBool[k] = true - } else { - ret.TagsBool[k] = false - } - } - if span.Method != nil { - ret.Method = *span.Method + return &dao.Span{ + TraceID: span.TraceID, + SpanID: span.SpanID, + SpaceID: span.SpaceID, + SpanType: span.SpanType, + SpanName: span.SpanName, + ParentID: span.ParentID, + Method: span.Method, + Psm: span.Psm, + Logid: span.Logid, + StartTime: span.StartTime, // us + CallType: span.CallType, + Duration: span.Duration, + StatusCode: span.StatusCode, + ObjectStorage: span.ObjectStorage, + Input: span.Input, + Output: span.Output, + LogicDeleteDate: span.LogicDeleteDate, + ReserveCreateTime: span.ReserveCreateTime, + TagsBool: span.TagsBool, + TagsFloat: span.TagsFloat, + TagsString: span.TagsString, + TagsLong: span.TagsLong, + TagsByte: span.TagsByte, + SystemTagsFloat: span.SystemTagsFloat, + SystemTagsLong: span.SystemTagsLong, + SystemTagsString: span.SystemTagsString, } - if span.CallType != nil { - ret.CallType = *span.CallType - } - if span.ObjectStorage != nil { - ret.ObjectStorage = *span.ObjectStorage - } - if span.Psm != nil { - ret.PSM = *span.Psm - } - if span.Logid != nil { - ret.LogID = *span.Logid - } - return ret -} - -func CopyMap[T any](in map[string]T) map[string]T { - ret := make(map[string]T) - for k, v := range in { - ret[k] = v - } - return ret } diff --git a/backend/modules/observability/infra/repo/ck/convertor/span_test.go b/backend/modules/observability/infra/repo/ck/convertor/span_test.go new file mode 100644 index 000000000..379ca3c24 --- /dev/null +++ b/backend/modules/observability/infra/repo/ck/convertor/span_test.go @@ -0,0 +1,600 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 + +package convertor + +import ( + "testing" + + "github.com/coze-dev/coze-loop/backend/modules/observability/infra/repo/ck/gorm_gen/model" + "github.com/coze-dev/coze-loop/backend/modules/observability/infra/repo/dao" + "github.com/stretchr/testify/assert" +) + +func TestSpanListPO2CKModels(t *testing.T) { + method1 := "POST" + psm1 := "service-1" + logid1 := "log-1" + callType1 := "http" + objectStorage1 := "tos://bucket/key1" + reserveCreateTime1 := "2222222222" + method2 := "GET" + psm2 := "service-2" + logid2 := "log-2" + callType2 := "grpc" + objectStorage2 := "tos://bucket/key2" + + tests := []struct { + name string + spans []*dao.Span + }{ + { + name: "multiple spans conversion", + spans: []*dao.Span{ + { + TraceID: "trace-1", + SpanID: "span-1", + SpaceID: "space-1", + SpanType: "model", + SpanName: "test-span-1", + ParentID: "parent-1", + Method: &method1, + Psm: &psm1, + Logid: &logid1, + StartTime: 1234567890, + CallType: &callType1, + Duration: 987654321, + StatusCode: 0, + ObjectStorage: &objectStorage1, + LogicDeleteDate: 1111111111, + ReserveCreateTime: &reserveCreateTime1, + TagsBool: map[string]uint8{ + "bool1": 1, + }, + TagsFloat: map[string]float64{ + "float1": 1.23, + }, + TagsString: map[string]string{ + "str1": "value1", + }, + TagsLong: map[string]int64{ + "long1": 123, + }, + TagsByte: map[string]string{ + "bytes1": "0101", + }, + SystemTagsFloat: map[string]float64{ + "sys_float1": 4.56, + }, + SystemTagsLong: map[string]int64{ + "sys_long1": 456, + }, + SystemTagsString: map[string]string{ + "sys_str1": "sys_value1", + }, + }, + { + TraceID: "trace-2", + SpanID: "span-2", + SpaceID: "space-2", + SpanType: "prompt", + SpanName: "test-span-2", + ParentID: "parent-2", + Method: &method2, + Psm: &psm2, + Logid: &logid2, + StartTime: 1234567891, + CallType: &callType2, + Duration: 987654322, + StatusCode: 1, + ObjectStorage: &objectStorage2, + TagsBool: map[string]uint8{ + "bool2": 0, + }, + TagsFloat: map[string]float64{ + "float2": 2.34, + }, + TagsString: map[string]string{ + "str2": "value2", + }, + TagsLong: map[string]int64{ + "long2": 234, + }, + TagsByte: map[string]string{ + "bytes2": "1010", + }, + SystemTagsFloat: map[string]float64{ + "sys_float2": 5.67, + }, + SystemTagsLong: map[string]int64{ + "sys_long2": 567, + }, + SystemTagsString: map[string]string{ + "sys_str2": "sys_value2", + }, + }, + }, + }, + { + name: "empty spans list", + spans: []*dao.Span{}, + }, + { + name: "nil spans list", + spans: nil, + }, + { + name: "spans with nil elements", + spans: []*dao.Span{ + { + SpanID: "valid-span", + TraceID: "valid-trace", + SpaceID: "valid-space", + SpanName: "valid-name", + StartTime: 1234567890, + }, + nil, + { + SpanID: "another-valid", + TraceID: "another-trace", + SpaceID: "another-space", + SpanName: "another-name", + StartTime: 1234567891, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := SpanListPO2CKModels(tt.spans) + + if tt.spans == nil { + assert.NotNil(t, result) + assert.Len(t, result, 0) + return + } + + assert.NotNil(t, result) + assert.Len(t, result, len(tt.spans)) + + for i, span := range tt.spans { + if span == nil { + // SpanPO2CKModel 会处理 nil，返回 nil + assert.Nil(t, result[i]) + } else { + assert.NotNil(t, result[i]) + assert.Equal(t, span.TraceID, result[i].TraceID) + assert.Equal(t, span.SpanID, result[i].SpanID) + assert.Equal(t, span.SpaceID, result[i].SpaceID) + assert.Equal(t, span.SpanType, result[i].SpanType) + assert.Equal(t, span.SpanName, result[i].SpanName) + assert.Equal(t, span.ParentID, result[i].ParentID) + assert.Equal(t, span.Method, result[i].Method) + assert.Equal(t, span.Psm, result[i].Psm) + assert.Equal(t, span.Logid, result[i].Logid) + assert.Equal(t, span.StartTime, result[i].StartTime) + assert.Equal(t, span.CallType, result[i].CallType) + assert.Equal(t, span.Duration, result[i].Duration) + assert.Equal(t, span.StatusCode, result[i].StatusCode) + assert.Equal(t, span.ObjectStorage, result[i].ObjectStorage) + assert.Equal(t, span.LogicDeleteDate, result[i].LogicDeleteDate) + assert.Equal(t, span.ReserveCreateTime, result[i].ReserveCreateTime) + assert.Equal(t, span.TagsBool, result[i].TagsBool) + assert.Equal(t, span.TagsFloat, result[i].TagsFloat) + assert.Equal(t, span.TagsString, result[i].TagsString) + assert.Equal(t, span.TagsLong, result[i].TagsLong) + assert.Equal(t, span.TagsByte, result[i].TagsByte) + assert.Equal(t, span.SystemTagsFloat, result[i].SystemTagsFloat) + assert.Equal(t, span.SystemTagsLong, result[i].SystemTagsLong) + assert.Equal(t, span.SystemTagsString, result[i].SystemTagsString) + } + } + }) + } +} + +func TestSpanListCKModels2PO(t *testing.T) { + method1 := "POST" + psm1 := "service-1" + logid1 := "log-1" + callType1 := "http" + objectStorage1 := "tos://bucket/key1" + reserveCreateTime1 := "2222222222" + method2 := "GET" + psm2 := "service-2" + logid2 := "log-2" + callType2 := "grpc" + objectStorage2 := "tos://bucket/key2" + + tests := []struct { + name string + spans []*model.ObservabilitySpan + }{ + { + name: "multiple ck models conversion", + spans: []*model.ObservabilitySpan{ + { + TraceID: "trace-1", + SpanID: "span-1", + SpaceID: "space-1", + SpanType: "model", + SpanName: "test-span-1", + ParentID: "parent-1", + Method: &method1, + Psm: &psm1, + Logid: &logid1, + StartTime: 1234567890, + CallType: &callType1, + Duration: 987654321, + StatusCode: 0, + ObjectStorage: &objectStorage1, + LogicDeleteDate: 1111111111, + ReserveCreateTime: &reserveCreateTime1, + TagsBool: map[string]uint8{ + "bool1": 1, + }, + TagsFloat: map[string]float64{ + "float1": 1.23, + }, + TagsString: map[string]string{ + "str1": "value1", + }, + TagsLong: map[string]int64{ + "long1": 123, + }, + TagsByte: map[string]string{ + "bytes1": "0101", + }, + SystemTagsFloat: map[string]float64{ + "sys_float1": 4.56, + }, + SystemTagsLong: map[string]int64{ + "sys_long1": 456, + }, + SystemTagsString: map[string]string{ + "sys_str1": "sys_value1", + }, + }, + { + TraceID: "trace-2", + SpanID: "span-2", + SpaceID: "space-2", + SpanType: "prompt", + SpanName: "test-span-2", + ParentID: "parent-2", + Method: &method2, + Psm: &psm2, + Logid: &logid2, + StartTime: 1234567891, + CallType: &callType2, + Duration: 987654322, + StatusCode: 1, + ObjectStorage: &objectStorage2, + TagsBool: map[string]uint8{ + "bool2": 0, + }, + TagsFloat: map[string]float64{ + "float2": 2.34, + }, + TagsString: map[string]string{ + "str2": "value2", + }, + TagsLong: map[string]int64{ + "long2": 234, + }, + TagsByte: map[string]string{ + "bytes2": "1010", + }, + SystemTagsFloat: map[string]float64{ + "sys_float2": 5.67, + }, + SystemTagsLong: map[string]int64{ + "sys_long2": 567, + }, + SystemTagsString: map[string]string{ + "sys_str2": "sys_value2", + }, + }, + }, + }, + { + name: "empty ck models list", + spans: []*model.ObservabilitySpan{}, + }, + { + name: "nil ck models list", + spans: nil, + }, + { + name: "ck models with nil elements", + spans: []*model.ObservabilitySpan{ + { + SpanID: "valid-span", + TraceID: "valid-trace", + SpaceID: "valid-space", + SpanName: "valid-name", + StartTime: 1234567890, + }, + nil, + { + SpanID: "another-valid", + TraceID: "another-trace", + SpaceID: "another-space", + SpanName: "another-name", + StartTime: 1234567891, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := SpanListCKModels2PO(tt.spans) + + if tt.spans == nil { + assert.NotNil(t, result) + assert.Len(t, result, 0) + return + } + + assert.NotNil(t, result) + assert.Len(t, result, len(tt.spans)) + + for i, span := range tt.spans { + if span == nil { + // SpanCKModel2PO 会处理 nil，返回 nil + assert.Nil(t, result[i]) + } else { + assert.NotNil(t, result[i]) + assert.Equal(t, span.TraceID, result[i].TraceID) + assert.Equal(t, span.SpanID, result[i].SpanID) + assert.Equal(t, span.SpaceID, result[i].SpaceID) + assert.Equal(t, span.SpanType, result[i].SpanType) + assert.Equal(t, span.SpanName, result[i].SpanName) + assert.Equal(t, span.ParentID, result[i].ParentID) + assert.Equal(t, span.Method, result[i].Method) + assert.Equal(t, span.Psm, result[i].Psm) + assert.Equal(t, span.Logid, result[i].Logid) + assert.Equal(t, span.StartTime, result[i].StartTime) + assert.Equal(t, span.CallType, result[i].CallType) + assert.Equal(t, span.Duration, result[i].Duration) + assert.Equal(t, span.StatusCode, result[i].StatusCode) + assert.Equal(t, span.ObjectStorage, result[i].ObjectStorage) + assert.Equal(t, span.LogicDeleteDate, result[i].LogicDeleteDate) + assert.Equal(t, span.ReserveCreateTime, result[i].ReserveCreateTime) + assert.Equal(t, span.TagsBool, result[i].TagsBool) + assert.Equal(t, span.TagsFloat, result[i].TagsFloat) + assert.Equal(t, span.TagsString, result[i].TagsString) + assert.Equal(t, span.TagsLong, result[i].TagsLong) + assert.Equal(t, span.TagsByte, result[i].TagsByte) + assert.Equal(t, span.SystemTagsFloat, result[i].SystemTagsFloat) + assert.Equal(t, span.SystemTagsLong, result[i].SystemTagsLong) + assert.Equal(t, span.SystemTagsString, result[i].SystemTagsString) + } + } + }) + } +} + +func TestSpanPO2CKModel(t *testing.T) { + method := "POST" + psm := "test-service" + logid := "log-456" + callType := "http" + objectStorage := "tos://bucket/key" + reserveCreateTime := "2222222222" + + tests := []struct { + name string + span *dao.Span + }{ + { + name: "complete span conversion", + span: &dao.Span{ + TraceID: "trace-123", + SpanID: "span-456", + SpaceID: "space-789", + SpanType: "model", + SpanName: "test-span", + ParentID: "parent-123", + Method: &method, + Psm: &psm, + Logid: &logid, + StartTime: 1234567890, + CallType: &callType, + Duration: 987654321, + StatusCode: 0, + ObjectStorage: &objectStorage, + LogicDeleteDate: 1111111111, + ReserveCreateTime: &reserveCreateTime, + TagsBool: map[string]uint8{ + "bool_key": 1, + }, + TagsFloat: map[string]float64{ + "float_key": 1.23, + }, + TagsString: map[string]string{ + "str_key": "str_value", + }, + TagsLong: map[string]int64{ + "long_key": 123, + }, + TagsByte: map[string]string{ + "bytes_key": "0101", + }, + SystemTagsFloat: map[string]float64{ + "sys_float_key": 4.56, + }, + SystemTagsLong: map[string]int64{ + "sys_long_key": 456, + }, + SystemTagsString: map[string]string{ + "sys_str_key": "sys_str_value", + }, + }, + }, + { + name: "minimal span conversion", + span: &dao.Span{ + SpanID: "minimal", + TraceID: "trace-min", + SpaceID: "space-min", + SpanName: "minimal-span", + StartTime: 0, + }, + }, + { + name: "nil span", + span: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := SpanPO2CKModel(tt.span) + + if tt.span == nil { + assert.Nil(t, result) + return + } + + assert.NotNil(t, result) + assert.Equal(t, tt.span.TraceID, result.TraceID) + assert.Equal(t, tt.span.SpanID, result.SpanID) + assert.Equal(t, tt.span.SpaceID, result.SpaceID) + assert.Equal(t, tt.span.SpanType, result.SpanType) + assert.Equal(t, tt.span.SpanName, result.SpanName) + assert.Equal(t, tt.span.ParentID, result.ParentID) + assert.Equal(t, tt.span.Method, result.Method) + assert.Equal(t, tt.span.Psm, result.Psm) + assert.Equal(t, tt.span.Logid, result.Logid) + assert.Equal(t, tt.span.StartTime, result.StartTime) + assert.Equal(t, tt.span.CallType, result.CallType) + assert.Equal(t, tt.span.Duration, result.Duration) + assert.Equal(t, tt.span.StatusCode, result.StatusCode) + assert.Equal(t, tt.span.ObjectStorage, result.ObjectStorage) + assert.Equal(t, tt.span.LogicDeleteDate, result.LogicDeleteDate) + assert.Equal(t, tt.span.ReserveCreateTime, result.ReserveCreateTime) + assert.Equal(t, tt.span.TagsBool, result.TagsBool) + assert.Equal(t, tt.span.TagsFloat, result.TagsFloat) + assert.Equal(t, tt.span.TagsString, result.TagsString) + assert.Equal(t, tt.span.TagsLong, result.TagsLong) + assert.Equal(t, tt.span.TagsByte, result.TagsByte) + assert.Equal(t, tt.span.SystemTagsFloat, result.SystemTagsFloat) + assert.Equal(t, tt.span.SystemTagsLong, result.SystemTagsLong) + assert.Equal(t, tt.span.SystemTagsString, result.SystemTagsString) + }) + } +} + +func TestSpanCKModel2PO(t *testing.T) { + method := "POST" + psm := "test-service" + logid := "log-456" + callType := "http" + objectStorage := "tos://bucket/key" + reserveCreateTime := "2222222222" + + tests := []struct { + name string + span *model.ObservabilitySpan + }{ + { + name: "complete ck model conversion", + span: &model.ObservabilitySpan{ + TraceID: "trace-123", + SpanID: "span-456", + SpaceID: "space-789", + SpanType: "model", + SpanName: "test-span", + ParentID: "parent-123", + Method: &method, + Psm: &psm, + Logid: &logid, + StartTime: 1234567890, + CallType: &callType, + Duration: 987654321, + StatusCode: 0, + ObjectStorage: &objectStorage, + LogicDeleteDate: 1111111111, + ReserveCreateTime: &reserveCreateTime, + TagsBool: map[string]uint8{ + "bool_key": 1, + }, + TagsFloat: map[string]float64{ + "float_key": 1.23, + }, + TagsString: map[string]string{ + "str_key": "str_value", + }, + TagsLong: map[string]int64{ + "long_key": 123, + }, + TagsByte: map[string]string{ + "bytes_key": "0101", + }, + SystemTagsFloat: map[string]float64{ + "sys_float_key": 4.56, + }, + SystemTagsLong: map[string]int64{ + "sys_long_key": 456, + }, + SystemTagsString: map[string]string{ + "sys_str_key": "sys_str_value", + }, + }, + }, + { + name: "minimal ck model conversion", + span: &model.ObservabilitySpan{ + SpanID: "minimal", + TraceID: "trace-min", + SpaceID: "space-min", + SpanName: "minimal-span", + StartTime: 0, + }, + }, + { + name: "nil ck model", + span: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := SpanCKModel2PO(tt.span) + + if tt.span == nil { + assert.Nil(t, result) + return + } + + assert.NotNil(t, result) + assert.Equal(t, tt.span.TraceID, result.TraceID) + assert.Equal(t, tt.span.SpanID, result.SpanID) + assert.Equal(t, tt.span.SpaceID, result.SpaceID) + assert.Equal(t, tt.span.SpanType, result.SpanType) + assert.Equal(t, tt.span.SpanName, result.SpanName) + assert.Equal(t, tt.span.ParentID, result.ParentID) + assert.Equal(t, tt.span.Method, result.Method) + assert.Equal(t, tt.span.Psm, result.Psm) + assert.Equal(t, tt.span.Logid, result.Logid) + assert.Equal(t, tt.span.StartTime, result.StartTime) + assert.Equal(t, tt.span.CallType, result.CallType) + assert.Equal(t, tt.span.Duration, result.Duration) + assert.Equal(t, tt.span.StatusCode, result.StatusCode) + assert.Equal(t, tt.span.ObjectStorage, result.ObjectStorage) + assert.Equal(t, tt.span.LogicDeleteDate, result.LogicDeleteDate) + assert.Equal(t, tt.span.ReserveCreateTime, result.ReserveCreateTime) + assert.Equal(t, tt.span.TagsBool, result.TagsBool) + assert.Equal(t, tt.span.TagsFloat, result.TagsFloat) + assert.Equal(t, tt.span.TagsString, result.TagsString) + assert.Equal(t, tt.span.TagsLong, result.TagsLong) + assert.Equal(t, tt.span.TagsByte, result.TagsByte) + assert.Equal(t, tt.span.SystemTagsFloat, result.SystemTagsFloat) + assert.Equal(t, tt.span.SystemTagsLong, result.SystemTagsLong) + assert.Equal(t, tt.span.SystemTagsString, result.SystemTagsString) + }) + } +} diff --git a/backend/modules/observability/infra/repo/ck/mocks/annotation_dao.go b/backend/modules/observability/infra/repo/ck/mocks/annotation_dao.go index a81ff6b91..0110c2a17 100644 --- a/backend/modules/observability/infra/repo/ck/mocks/annotation_dao.go +++ b/backend/modules/observability/infra/repo/ck/mocks/annotation_dao.go @@ -13,8 +13,7 @@ import ( context "context" reflect "reflect" - ck "github.com/coze-dev/coze-loop/backend/modules/observability/infra/repo/ck" - model "github.com/coze-dev/coze-loop/backend/modules/observability/infra/repo/ck/gorm_gen/model" + dao "github.com/coze-dev/coze-loop/backend/modules/observability/infra/repo/dao" gomock "go.uber.org/mock/gomock" ) @@ -43,10 +42,10 @@ func (m *MockIAnnotationDao) EXPECT() *MockIAnnotationDaoMockRecorder { } // Get mocks base method. -func (m *MockIAnnotationDao) Get(arg0 context.Context, arg1 *ck.GetAnnotationParam) (*model.ObservabilityAnnotation, error) { +func (m *MockIAnnotationDao) Get(arg0 context.Context, arg1 *dao.GetAnnotationParam) (*dao.Annotation, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Get", arg0, arg1) - ret0, _ := ret[0].(*model.ObservabilityAnnotation) + ret0, _ := ret[0].(*dao.Annotation) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -58,7 +57,7 @@ func (mr *MockIAnnotationDaoMockRecorder) Get(arg0, arg1 any) *gomock.Call { } // Insert mocks base method. -func (m *MockIAnnotationDao) Insert(arg0 context.Context, arg1 *ck.InsertAnnotationParam) error { +func (m *MockIAnnotationDao) Insert(arg0 context.Context, arg1 *dao.InsertAnnotationParam) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Insert", arg0, arg1) ret0, _ := ret[0].(error) @@ -72,10 +71,10 @@ func (mr *MockIAnnotationDaoMockRecorder) Insert(arg0, arg1 any) *gomock.Call { } // List mocks base method. -func (m *MockIAnnotationDao) List(arg0 context.Context, arg1 *ck.ListAnnotationsParam) ([]*model.ObservabilityAnnotation, error) { +func (m *MockIAnnotationDao) List(arg0 context.Context, arg1 *dao.ListAnnotationsParam) ([]*dao.Annotation, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "List", arg0, arg1) - ret0, _ := ret[0].([]*model.ObservabilityAnnotation) + ret0, _ := ret[0].([]*dao.Annotation) ret1, _ := ret[1].(error) return ret0, ret1 } diff --git a/backend/modules/observability/infra/repo/ck/mocks/spans_dao.go b/backend/modules/observability/infra/repo/ck/mocks/spans_dao.go index 20a99c663..6532611ba 100644 --- a/backend/modules/observability/infra/repo/ck/mocks/spans_dao.go +++ b/backend/modules/observability/infra/repo/ck/mocks/spans_dao.go @@ -13,8 +13,7 @@ import ( context "context" reflect "reflect" - ck "github.com/coze-dev/coze-loop/backend/modules/observability/infra/repo/ck" - model "github.com/coze-dev/coze-loop/backend/modules/observability/infra/repo/ck/gorm_gen/model" + dao "github.com/coze-dev/coze-loop/backend/modules/observability/infra/repo/dao" gomock "go.uber.org/mock/gomock" ) @@ -43,10 +42,10 @@ func (m *MockISpansDao) EXPECT() *MockISpansDaoMockRecorder { } // Get mocks base method. -func (m *MockISpansDao) Get(arg0 context.Context, arg1 *ck.QueryParam) ([]*model.ObservabilitySpan, error) { +func (m *MockISpansDao) Get(arg0 context.Context, arg1 *dao.QueryParam) ([]*dao.Span, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Get", arg0, arg1) - ret0, _ := ret[0].([]*model.ObservabilitySpan) + ret0, _ := ret[0].([]*dao.Span) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -58,7 +57,7 @@ func (mr *MockISpansDaoMockRecorder) Get(arg0, arg1 any) *gomock.Call { } // GetMetrics mocks base method. -func (m *MockISpansDao) GetMetrics(ctx context.Context, param *ck.GetMetricsParam) ([]map[string]any, error) { +func (m *MockISpansDao) GetMetrics(ctx context.Context, param *dao.GetMetricsParam) ([]map[string]any, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetMetrics", ctx, param) ret0, _ := ret[0].([]map[string]any) @@ -73,7 +72,7 @@ func (mr *MockISpansDaoMockRecorder) GetMetrics(ctx, param any) *gomock.Call { } // Insert mocks base method. -func (m *MockISpansDao) Insert(arg0 context.Context, arg1 *ck.InsertParam) error { +func (m *MockISpansDao) Insert(arg0 context.Context, arg1 *dao.InsertParam) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Insert", arg0, arg1) ret0, _ := ret[0].(error) diff --git a/backend/modules/observability/infra/repo/ck/spans.go b/backend/modules/observability/infra/repo/ck/spans.go index 6953d04f4..ac228ea6c 100644 --- a/backend/modules/observability/infra/repo/ck/spans.go +++ b/backend/modules/observability/infra/repo/ck/spans.go @@ -7,6 +7,8 @@ import ( "bytes" "context" "fmt" + "github.com/coze-dev/coze-loop/backend/modules/observability/infra/repo/ck/convertor" + "github.com/coze-dev/coze-loop/backend/modules/observability/infra/repo/dao" "regexp" "strconv" "strings" @@ -24,47 +26,10 @@ import ( ) const ( - QueryTypeGetTrace = "get_trace" - QueryTypeListSpans = "list_spans" + TraceStorageTypeCK = "ck" ) -type QueryParam struct { - QueryType string // for sql optimization - Tables []string - AnnoTableMap map[string]string - StartTime int64 // us - EndTime int64 // us - Filters *loop_span.FilterFields - Limit int32 - OrderByStartTime bool - SelectColumns []string - OmitColumns []string // omit specific columns -} - -type InsertParam struct { - Table string - Spans []*model.ObservabilitySpan -} - -//go:generate mockgen -destination=mocks/spans_dao.go -package=mocks . ISpansDao -type ISpansDao interface { - Insert(context.Context, *InsertParam) error - Get(context.Context, *QueryParam) ([]*model.ObservabilitySpan, error) - GetMetrics(ctx context.Context, param *GetMetricsParam) ([]map[string]any, error) -} - -// GetMetricsParam 指标查询参数 -type GetMetricsParam struct { - Tables []string - Aggregations []*metrics_entity.Dimension - GroupBys []*metrics_entity.Dimension - Filters *loop_span.FilterFields - StartAt int64 - EndAt int64 - Granularity metrics_entity.MetricGranularity -} - -func NewSpansCkDaoImpl(db ck.Provider) (ISpansDao, error) { +func NewSpansCkDaoImpl(db ck.Provider) (dao.ISpansDao, error) { return &SpansCkDaoImpl{ db: db, }, nil @@ -78,16 +43,17 @@ func (s *SpansCkDaoImpl) newSession(ctx context.Context) *gorm.DB { return s.db.NewSession(ctx) } -func (s *SpansCkDaoImpl) Insert(ctx context.Context, param *InsertParam) error { +func (s *SpansCkDaoImpl) Insert(ctx context.Context, param *dao.InsertParam) error { db := s.newSession(ctx) retryTimes := 3 var lastErr error // 满足条件的批写入会保证幂等性； // 如果是网络问题导致错误, 重试可能会导致重复写入; // https://clickhouse.com/docs/guides/developer/transactional。 + spans := convertor.SpanListPO2CKModels(param.Spans) for i := 0; i < retryTimes; i++ { - if err := db.Table(param.Table).Create(param.Spans).Error; err != nil { - logs.CtxError(ctx, "fail to insert spans, count %d, %v", len(param.Spans), err) + if err := db.Table(param.Table).Create(spans).Error; err != nil { + logs.CtxError(ctx, "fail to insert spans, count %d, %v", len(spans), err) lastErr = err } else { return nil @@ -96,7 +62,7 @@ func (s *SpansCkDaoImpl) Insert(ctx context.Context, param *InsertParam) error { return lastErr } -func (s *SpansCkDaoImpl) Get(ctx context.Context, param *QueryParam) ([]*model.ObservabilitySpan, error) { +func (s *SpansCkDaoImpl) Get(ctx context.Context, param *dao.QueryParam) ([]*dao.Span, error) { sql, err := s.buildSql(ctx, param) if err != nil { return nil, errorx.WrapByCode(err, obErrorx.CommercialCommonInvalidParamCodeCode, errorx.WithExtraMsg("invalid get trace request")) @@ -108,13 +74,13 @@ func (s *SpansCkDaoImpl) Get(ctx context.Context, param *QueryParam) ([]*model.O if err := sql.Find(&spans).Error; err != nil { return nil, errorx.WrapByCode(err, obErrorx.CommercialCommonRPCErrorCodeCode) } - return spans, nil + return convertor.SpanListCKModels2PO(spans), nil } // select/inner_query/group_by/order_by/with_fill var metricsSqlTemplate = `SELECT %s FROM (%s) %s %s` -func (s *SpansCkDaoImpl) GetMetrics(ctx context.Context, param *GetMetricsParam) ([]map[string]any, error) { +func (s *SpansCkDaoImpl) GetMetrics(ctx context.Context, param *dao.GetMetricsParam) ([]map[string]any, error) { sql, err := s.buildMetricsSql(ctx, param) if err != nil { return nil, err @@ -128,9 +94,9 @@ func (s *SpansCkDaoImpl) GetMetrics(ctx context.Context, param *GetMetricsParam) return result, nil } -func (s *SpansCkDaoImpl) buildMetricsSql(ctx context.Context, param *GetMetricsParam) (string, error) { +func (s *SpansCkDaoImpl) buildMetricsSql(ctx context.Context, param *dao.GetMetricsParam) (string, error) { // 直接复用现有的SQL获取所有数据, 然后再计算指标 - sql, err := s.buildSql(ctx, &QueryParam{ + sql, err := s.buildSql(ctx, &dao.QueryParam{ Tables: param.Tables, StartTime: param.StartAt, EndTime: param.EndAt, @@ -204,7 +170,7 @@ func (s *SpansCkDaoImpl) formatAggregationExpression(ctx context.Context, dimens return fmt.Sprintf(dimension.Expression.Expression, replacements...), nil } -func (s *SpansCkDaoImpl) buildSql(ctx context.Context, param *QueryParam) (*gorm.DB, error) { +func (s *SpansCkDaoImpl) buildSql(ctx context.Context, param *dao.QueryParam) (*gorm.DB, error) { db := s.newSession(ctx) var tableQueries []*gorm.DB for _, table := range param.Tables { @@ -237,7 +203,7 @@ func (s *SpansCkDaoImpl) buildSql(ctx context.Context, param *QueryParam) (*gorm } } -func (s *SpansCkDaoImpl) buildSingleSql(ctx context.Context, db *gorm.DB, tableName string, param *QueryParam) (*gorm.DB, error) { +func (s *SpansCkDaoImpl) buildSingleSql(ctx context.Context, db *gorm.DB, tableName string, param *dao.QueryParam) (*gorm.DB, error) { sqlQuery, err := s.buildSqlForFilterFields(ctx, db, param.Filters) if err != nil { return nil, err diff --git a/backend/modules/observability/infra/repo/ck/spans_test.go b/backend/modules/observability/infra/repo/ck/spans_test.go index 382be3833..937b964c0 100644 --- a/backend/modules/observability/infra/repo/ck/spans_test.go +++ b/backend/modules/observability/infra/repo/ck/spans_test.go @@ -10,6 +10,7 @@ import ( "github.com/DATA-DOG/go-sqlmock" "github.com/coze-dev/coze-loop/backend/modules/observability/domain/trace/entity/loop_span" "github.com/coze-dev/coze-loop/backend/modules/observability/infra/repo/ck/gorm_gen/model" + "github.com/coze-dev/coze-loop/backend/modules/observability/infra/repo/dao" "github.com/coze-dev/coze-loop/backend/pkg/lang/ptr" "github.com/stretchr/testify/assert" "gorm.io/driver/clickhouse" @@ -382,7 +383,7 @@ func TestBuildSql(t *testing.T) { }, } for _, tc := range testCases { - qDb, err := new(SpansCkDaoImpl).buildSingleSql(context.Background(), db, "observability_spans", &QueryParam{ + qDb, err := new(SpansCkDaoImpl).buildSingleSql(context.Background(), db, "observability_spans", &dao.QueryParam{ StartTime: 1, EndTime: 2, Filters: tc.filter, @@ -566,7 +567,7 @@ func TestQueryTypeEnumNotMatchSqlExceptionCases(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - qDb, err := new(SpansCkDaoImpl).buildSingleSql(context.Background(), db, "observability_spans", &QueryParam{ + qDb, err := new(SpansCkDaoImpl).buildSingleSql(context.Background(), db, "observability_spans", &dao.QueryParam{ StartTime: 1, EndTime: 2, Filters: tc.filter, @@ -713,7 +714,7 @@ func TestQueryTypeEnumNotMatchComplexScenarios(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - qDb, err := new(SpansCkDaoImpl).buildSingleSql(context.Background(), db, "observability_spans", &QueryParam{ + qDb, err := new(SpansCkDaoImpl).buildSingleSql(context.Background(), db, "observability_spans", &dao.QueryParam{ StartTime: 1, EndTime: 2, Filters: tc.filter, diff --git a/backend/modules/observability/infra/repo/dao/annotation.go b/backend/modules/observability/infra/repo/dao/annotation.go new file mode 100644 index 000000000..a4705b437 --- /dev/null +++ b/backend/modules/observability/infra/repo/dao/annotation.go @@ -0,0 +1,65 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 +package dao + +import ( + "context" +) + +type InsertAnnotationParam struct { + Table string + Annotations []*Annotation + Extra map[string]string +} + +type GetAnnotationParam struct { + Tables []string + ID string + StartTime int64 // us + EndTime int64 // us + Limit int32 + Extra map[string]string +} + +type ListAnnotationsParam struct { + Tables []string + SpanIDs []string + StartTime int64 // us + EndTime int64 // us + DescByUpdatedAt bool + Limit int32 + Extra map[string]string +} + +//go:generate mockgen -destination=mocks/annotation_dao.go -package=mocks . IAnnotationDao +type IAnnotationDao interface { + Insert(context.Context, *InsertAnnotationParam) error + Get(context.Context, *GetAnnotationParam) (*Annotation, error) + List(context.Context, *ListAnnotationsParam) ([]*Annotation, error) +} + +type Annotation struct { + ID string `json:"id"` + SpanID string `json:"span_id"` + TraceID string `json:"trace_id"` + StartTime int64 `json:"start_time"` + SpaceID string `json:"space_id"` + AnnotationType string `json:"annotation_type"` + AnnotationIndex []string `json:"annotation_index"` + Key string `json:"key"` + ValueType string `json:"value_type"` + ValueString string `json:"value_string"` + ValueLong int64 `json:"value_long"` + ValueFloat float64 `json:"value_float"` + ValueBool bool `json:"value_bool"` + Reasoning string `json:"reasoning"` + Correction string `json:"correction"` + Metadata string `json:"metadata"` + Status string `json:"status"` + CreatedBy string `json:"created_by"` + CreatedAt uint64 `json:"created_at"` + UpdatedBy string `json:"updated_by"` + UpdatedAt uint64 `json:"updated_at"` + DeletedAt uint64 `json:"deleted_at"` + StartDate string `json:"start_date"` +} diff --git a/backend/modules/observability/infra/repo/ck/convertor/annotation.go b/backend/modules/observability/infra/repo/dao/converter/annotation.go similarity index 92% rename from backend/modules/observability/infra/repo/ck/convertor/annotation.go rename to backend/modules/observability/infra/repo/dao/converter/annotation.go index 5cc0acc3f..6ba6737ce 100644 --- a/backend/modules/observability/infra/repo/ck/convertor/annotation.go +++ b/backend/modules/observability/infra/repo/dao/converter/annotation.go @@ -1,20 +1,18 @@ // Copyright (c) 2025 coze-dev Authors // SPDX-License-Identifier: Apache-2.0 - -package convertor +package converter import ( "fmt" - "time" - "github.com/bytedance/sonic" "github.com/coze-dev/coze-loop/backend/modules/observability/domain/trace/entity/loop_span" - "github.com/coze-dev/coze-loop/backend/modules/observability/infra/repo/ck/gorm_gen/model" + "github.com/coze-dev/coze-loop/backend/modules/observability/infra/repo/dao" "github.com/coze-dev/coze-loop/backend/pkg/json" "github.com/coze-dev/coze-loop/backend/pkg/logs" + "time" ) -func AnnotationPO2DO(annotation *model.ObservabilityAnnotation) *loop_span.Annotation { +func AnnotationPO2DO(annotation *dao.Annotation) *loop_span.Annotation { if annotation == nil { return nil } @@ -82,8 +80,8 @@ func AnnotationPO2DO(annotation *model.ObservabilityAnnotation) *loop_span.Annot return ret } -func AnnotationDO2PO(annotation *loop_span.Annotation) (*model.ObservabilityAnnotation, error) { - ret := &model.ObservabilityAnnotation{ +func AnnotationDO2PO(annotation *loop_span.Annotation) (*dao.Annotation, error) { + ret := &dao.Annotation{ ID: annotation.ID, SpanID: annotation.SpanID, TraceID: annotation.TraceID, @@ -131,7 +129,7 @@ func AnnotationDO2PO(annotation *loop_span.Annotation) (*model.ObservabilityAnno return ret, nil } -func AnnotationListPO2DO(annotations []*model.ObservabilityAnnotation) loop_span.AnnotationList { +func AnnotationListPO2DO(annotations []*dao.Annotation) loop_span.AnnotationList { ret := make(loop_span.AnnotationList, len(annotations)) for i, annotation := range annotations { ret[i] = AnnotationPO2DO(annotation) diff --git a/backend/modules/observability/infra/repo/dao/converter/span.go b/backend/modules/observability/infra/repo/dao/converter/span.go new file mode 100644 index 000000000..2d7fbe17a --- /dev/null +++ b/backend/modules/observability/infra/repo/dao/converter/span.go @@ -0,0 +1,144 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 +package converter + +import ( + "github.com/coze-dev/coze-loop/backend/modules/observability/domain/trace/entity/loop_span" + "github.com/coze-dev/coze-loop/backend/modules/observability/infra/repo/dao" + "github.com/coze-dev/coze-loop/backend/pkg/lang/ptr" + "time" +) + +func SpanListDO2PO(spans loop_span.SpanList, TTL loop_span.TTL) []*dao.Span { + ret := make([]*dao.Span, len(spans)) + for i, span := range spans { + ret[i] = SpanDO2PO(span, TTL) + } + return ret +} + +func SpanListPO2DO(spans []*dao.Span) loop_span.SpanList { + ret := make(loop_span.SpanList, len(spans)) + for i, span := range spans { + ret[i] = SpanPO2DO(span) + } + return ret +} + +func SpanDO2PO(span *loop_span.Span, TTL loop_span.TTL) *dao.Span { + ret := &dao.Span{ + TraceID: span.TraceID, + SpanID: span.SpanID, + SpaceID: span.WorkspaceID, + SpanType: span.SpanType, + SpanName: span.SpanName, + ParentID: span.ParentID, + StartTime: span.StartTime, // us + Duration: span.DurationMicros, + Psm: ptr.Of(span.PSM), + Logid: ptr.Of(span.LogID), + StatusCode: span.StatusCode, + Input: span.Input, + Output: span.Output, + TagsFloat: CopyMap(span.TagsDouble), + TagsString: CopyMap(span.TagsString), + TagsLong: CopyMap(span.TagsLong), + TagsByte: CopyMap(span.TagsByte), + SystemTagsFloat: CopyMap(span.SystemTagsDouble), + SystemTagsLong: CopyMap(span.SystemTagsLong), + SystemTagsString: CopyMap(span.SystemTagsString), + } + ret.TagsBool = make(map[string]uint8) + for k, v := range span.TagsBool { + if v { + ret.TagsBool[k] = 1 + } else { + ret.TagsBool[k] = 0 + } + } + if span.Method != "" { + ret.Method = ptr.Of(span.Method) + } + if span.CallType != "" { + ret.CallType = ptr.Of(span.CallType) + } + if span.ObjectStorage != "" { + ret.ObjectStorage = ptr.Of(span.ObjectStorage) + } + switch TTL { + case loop_span.TTL3d: + ret.LogicDeleteDate = time.Now().Add(3 * 24 * time.Hour).UnixMicro() + case loop_span.TTL7d: + ret.LogicDeleteDate = time.Now().Add(7 * 24 * time.Hour).UnixMicro() + case loop_span.TTL30d: + ret.LogicDeleteDate = time.Now().Add(30 * 24 * time.Hour).UnixMicro() + case loop_span.TTL90d: + ret.LogicDeleteDate = time.Now().Add(90 * 24 * time.Hour).UnixMicro() + case loop_span.TTL180d: + ret.LogicDeleteDate = time.Now().Add(180 * 24 * time.Hour).UnixMicro() + case loop_span.TTL365d: + ret.LogicDeleteDate = time.Now().Add(365 * 24 * time.Hour).UnixMicro() + default: + ret.LogicDeleteDate = time.Now().Add(3 * 24 * time.Hour).UnixMicro() + } + return ret +} + +func SpanPO2DO(span *dao.Span) *loop_span.Span { + if span == nil { + return nil + } + ret := &loop_span.Span{ + TraceID: span.TraceID, + SpanID: span.SpanID, + WorkspaceID: span.SpaceID, + SpanType: span.SpanType, + SpanName: span.SpanName, + ParentID: span.ParentID, + StartTime: span.StartTime, // us + DurationMicros: span.Duration, + StatusCode: span.StatusCode, + Input: span.Input, + Output: span.Output, + TagsDouble: CopyMap(span.TagsFloat), + TagsString: CopyMap(span.TagsString), + TagsLong: CopyMap(span.TagsLong), + TagsByte: CopyMap(span.TagsByte), + SystemTagsDouble: CopyMap(span.SystemTagsFloat), + SystemTagsLong: CopyMap(span.SystemTagsLong), + SystemTagsString: CopyMap(span.SystemTagsString), + LogicDeleteTime: span.LogicDeleteDate, + } + ret.TagsBool = make(map[string]bool) + for k, v := range span.TagsBool { + if v > 0 { + ret.TagsBool[k] = true + } else { + ret.TagsBool[k] = false + } + } + if span.Method != nil { + ret.Method = *span.Method + } + if span.CallType != nil { + ret.CallType = *span.CallType + } + if span.ObjectStorage != nil { + ret.ObjectStorage = *span.ObjectStorage + } + if span.Psm != nil { + ret.PSM = *span.Psm + } + if span.Logid != nil { + ret.LogID = *span.Logid + } + return ret +} + +func CopyMap[T any](in map[string]T) map[string]T { + ret := make(map[string]T) + for k, v := range in { + ret[k] = v + } + return ret +} diff --git a/backend/modules/observability/infra/repo/dao/mocks/annotation_dao.go b/backend/modules/observability/infra/repo/dao/mocks/annotation_dao.go new file mode 100644 index 000000000..97ff1cfd9 --- /dev/null +++ b/backend/modules/observability/infra/repo/dao/mocks/annotation_dao.go @@ -0,0 +1,86 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/coze-dev/coze-loop/backend/modules/observability/infra/repo/dao (interfaces: IAnnotationDao) +// +// Generated by this command: +// +// mockgen -destination=mocks/annotation_dao.go -package=mocks . IAnnotationDao +// + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + reflect "reflect" + + dao "github.com/coze-dev/coze-loop/backend/modules/observability/infra/repo/dao" + gomock "go.uber.org/mock/gomock" +) + +// MockIAnnotationDao is a mock of IAnnotationDao interface. +type MockIAnnotationDao struct { + ctrl *gomock.Controller + recorder *MockIAnnotationDaoMockRecorder + isgomock struct{} +} + +// MockIAnnotationDaoMockRecorder is the mock recorder for MockIAnnotationDao. +type MockIAnnotationDaoMockRecorder struct { + mock *MockIAnnotationDao +} + +// NewMockIAnnotationDao creates a new mock instance. +func NewMockIAnnotationDao(ctrl *gomock.Controller) *MockIAnnotationDao { + mock := &MockIAnnotationDao{ctrl: ctrl} + mock.recorder = &MockIAnnotationDaoMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockIAnnotationDao) EXPECT() *MockIAnnotationDaoMockRecorder { + return m.recorder +} + +// Get mocks base method. +func (m *MockIAnnotationDao) Get(arg0 context.Context, arg1 *dao.GetAnnotationParam) (*dao.Annotation, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Get", arg0, arg1) + ret0, _ := ret[0].(*dao.Annotation) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Get indicates an expected call of Get. +func (mr *MockIAnnotationDaoMockRecorder) Get(arg0, arg1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockIAnnotationDao)(nil).Get), arg0, arg1) +} + +// Insert mocks base method. +func (m *MockIAnnotationDao) Insert(arg0 context.Context, arg1 *dao.InsertAnnotationParam) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Insert", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// Insert indicates an expected call of Insert. +func (mr *MockIAnnotationDaoMockRecorder) Insert(arg0, arg1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Insert", reflect.TypeOf((*MockIAnnotationDao)(nil).Insert), arg0, arg1) +} + +// List mocks base method. +func (m *MockIAnnotationDao) List(arg0 context.Context, arg1 *dao.ListAnnotationsParam) ([]*dao.Annotation, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "List", arg0, arg1) + ret0, _ := ret[0].([]*dao.Annotation) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// List indicates an expected call of List. +func (mr *MockIAnnotationDaoMockRecorder) List(arg0, arg1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "List", reflect.TypeOf((*MockIAnnotationDao)(nil).List), arg0, arg1) +} diff --git a/backend/modules/observability/infra/repo/dao/mocks/spans_dao.go b/backend/modules/observability/infra/repo/dao/mocks/spans_dao.go new file mode 100644 index 000000000..55ac2484b --- /dev/null +++ b/backend/modules/observability/infra/repo/dao/mocks/spans_dao.go @@ -0,0 +1,86 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/coze-dev/coze-loop/backend/modules/observability/infra/repo/dao (interfaces: ISpansDao) +// +// Generated by this command: +// +// mockgen -destination=mocks/spans_dao.go -package=mocks . ISpansDao +// + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + reflect "reflect" + + dao "github.com/coze-dev/coze-loop/backend/modules/observability/infra/repo/dao" + gomock "go.uber.org/mock/gomock" +) + +// MockISpansDao is a mock of ISpansDao interface. +type MockISpansDao struct { + ctrl *gomock.Controller + recorder *MockISpansDaoMockRecorder + isgomock struct{} +} + +// MockISpansDaoMockRecorder is the mock recorder for MockISpansDao. +type MockISpansDaoMockRecorder struct { + mock *MockISpansDao +} + +// NewMockISpansDao creates a new mock instance. +func NewMockISpansDao(ctrl *gomock.Controller) *MockISpansDao { + mock := &MockISpansDao{ctrl: ctrl} + mock.recorder = &MockISpansDaoMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockISpansDao) EXPECT() *MockISpansDaoMockRecorder { + return m.recorder +} + +// Get mocks base method. +func (m *MockISpansDao) Get(arg0 context.Context, arg1 *dao.QueryParam) ([]*dao.Span, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Get", arg0, arg1) + ret0, _ := ret[0].([]*dao.Span) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Get indicates an expected call of Get. +func (mr *MockISpansDaoMockRecorder) Get(arg0, arg1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockISpansDao)(nil).Get), arg0, arg1) +} + +// GetMetrics mocks base method. +func (m *MockISpansDao) GetMetrics(ctx context.Context, param *dao.GetMetricsParam) ([]map[string]any, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetMetrics", ctx, param) + ret0, _ := ret[0].([]map[string]any) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetMetrics indicates an expected call of GetMetrics. +func (mr *MockISpansDaoMockRecorder) GetMetrics(ctx, param any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetMetrics", reflect.TypeOf((*MockISpansDao)(nil).GetMetrics), ctx, param) +} + +// Insert mocks base method. +func (m *MockISpansDao) Insert(arg0 context.Context, arg1 *dao.InsertParam) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Insert", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// Insert indicates an expected call of Insert. +func (mr *MockISpansDaoMockRecorder) Insert(arg0, arg1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Insert", reflect.TypeOf((*MockISpansDao)(nil).Insert), arg0, arg1) +} diff --git a/backend/modules/observability/infra/repo/dao/span.go b/backend/modules/observability/infra/repo/dao/span.go new file mode 100644 index 000000000..68d86174f --- /dev/null +++ b/backend/modules/observability/infra/repo/dao/span.go @@ -0,0 +1,82 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 +package dao + +import ( + "context" + + metrics_entity "github.com/coze-dev/coze-loop/backend/modules/observability/domain/metric/entity" + "github.com/coze-dev/coze-loop/backend/modules/observability/domain/trace/entity/loop_span" +) + +const ( + QueryTypeGetTrace = "get_trace" + QueryTypeListSpans = "list_spans" +) + +type QueryParam struct { + QueryType string // for sql optimization + Tables []string + AnnoTableMap map[string]string + StartTime int64 // us + EndTime int64 // us + Filters *loop_span.FilterFields + Limit int32 + OrderByStartTime bool + SelectColumns []string + OmitColumns []string // omit specific columns + Extra map[string]string +} + +type InsertParam struct { + Table string + Spans []*Span +} + +// GetMetricsParam 指标查询参数 +type GetMetricsParam struct { + Tables []string + Aggregations []*metrics_entity.Dimension + GroupBys []*metrics_entity.Dimension + Filters *loop_span.FilterFields + StartAt int64 + EndAt int64 + Granularity metrics_entity.MetricGranularity + Extra map[string]string +} + +//go:generate mockgen -destination=mocks/spans_dao.go -package=mocks . ISpansDao +type ISpansDao interface { + Insert(context.Context, *InsertParam) error + Get(context.Context, *QueryParam) ([]*Span, error) + GetMetrics(ctx context.Context, param *GetMetricsParam) ([]map[string]any, error) +} + +type Span struct { + TraceID string `json:"trace_id"` + SpanID string `json:"span_id"` + SpaceID string `json:"space_id"` + SpanType string `json:"span_type"` + SpanName string `json:"span_name"` + ParentID string `json:"parent_id"` + Method *string `json:"method"` + Psm *string `json:"psm"` + Logid *string `json:"logid"` + StartTime int64 `json:"start_time"` + CallType *string `json:"call_type"` + Duration int64 `json:"duration"` + StatusCode int32 `json:"status_code"` + ObjectStorage *string `json:"object_storage"` + Input string `json:"input"` + Output string `json:"output"` + LogicDeleteDate int64 `json:"logic_delete_date"` + ReserveCreateTime *string `json:"reserve_create_time"` + TagsBool map[string]uint8 `json:"tags_bool"` + TagsFloat map[string]float64 `json:"tags_float"` + TagsString map[string]string `json:"tags_string"` + TagsLong map[string]int64 `json:"tags_long"` + TagsByte map[string]string `json:"tags_byte"` + SystemTagsFloat map[string]float64 `json:"system_tags_float"` + SystemTagsLong map[string]int64 `json:"system_tags_long"` + SystemTagsString map[string]string `json:"system_tags_string"` +} diff --git a/backend/modules/observability/infra/repo/trace.go b/backend/modules/observability/infra/repo/trace.go index bc11e4604..f96ea75ff 100644 --- a/backend/modules/observability/infra/repo/trace.go +++ b/backend/modules/observability/infra/repo/trace.go @@ -6,20 +6,22 @@ package repo import ( "context" "encoding/base64" + "errors" "fmt" + "github.com/coze-dev/coze-loop/backend/modules/observability/domain/component/mq" + "github.com/coze-dev/coze-loop/backend/modules/observability/domain/component/storage" + "github.com/coze-dev/coze-loop/backend/modules/observability/domain/trace/entity" + "github.com/coze-dev/coze-loop/backend/modules/observability/infra/repo/dao" "strconv" "time" "github.com/coze-dev/coze-loop/backend/modules/observability/domain/component/config" - "github.com/coze-dev/coze-loop/backend/modules/observability/domain/component/mq" metric_repo "github.com/coze-dev/coze-loop/backend/modules/observability/domain/metric/repo" - "github.com/coze-dev/coze-loop/backend/modules/observability/domain/trace/entity" "github.com/coze-dev/coze-loop/backend/modules/observability/domain/trace/entity/loop_span" "github.com/coze-dev/coze-loop/backend/modules/observability/domain/trace/repo" "github.com/coze-dev/coze-loop/backend/modules/observability/infra/repo/ck" - "github.com/coze-dev/coze-loop/backend/modules/observability/infra/repo/ck/convertor" - "github.com/coze-dev/coze-loop/backend/modules/observability/infra/repo/ck/gorm_gen/model" - "github.com/coze-dev/coze-loop/backend/modules/observability/infra/repo/redis" + "github.com/coze-dev/coze-loop/backend/modules/observability/infra/repo/dao/converter" + redis_dao "github.com/coze-dev/coze-loop/backend/modules/observability/infra/repo/redis" obErrorx "github.com/coze-dev/coze-loop/backend/modules/observability/pkg/errno" "github.com/coze-dev/coze-loop/backend/pkg/errorx" "github.com/coze-dev/coze-loop/backend/pkg/json" @@ -29,43 +31,94 @@ import ( "github.com/samber/lo" ) -func NewTraceCKRepoImpl( - spanDao ck.ISpansDao, - annoDao ck.IAnnotationDao, +type TraceRepoOption func(*TraceRepoImpl) + +func WithTraceStorageDaos(storageType string, spanDao dao.ISpansDao, annoDao dao.IAnnotationDao) TraceRepoOption { + return func(t *TraceRepoImpl) { + WithTraceStorageSpanDao(storageType, spanDao)(t) + WithTraceStorageAnnotationDao(storageType, annoDao)(t) + } +} + +func WithTraceStorageSpanDao(storageType string, spanDao dao.ISpansDao) TraceRepoOption { + return func(t *TraceRepoImpl) { + if storageType == "" || spanDao == nil { + return + } + if t.spanDaos == nil { + t.spanDaos = make(map[string]dao.ISpansDao) + } + t.spanDaos[storageType] = spanDao + } +} + +func WithTraceStorageAnnotationDao(storageType string, annoDao dao.IAnnotationDao) TraceRepoOption { + return func(t *TraceRepoImpl) { + if storageType == "" || annoDao == nil { + return + } + if t.annoDaos == nil { + t.annoDaos = make(map[string]dao.IAnnotationDao) + } + t.annoDaos[storageType] = annoDao + } +} + +func NewTraceRepoImpl( traceConfig config.ITraceConfig, - spanRedisDao redis.ISpansRedisDao, + storageProvider storage.IStorageProvider, + spanRedisDao redis_dao.ISpansRedisDao, spanProducer mq.ISpanProducer, + opts ...TraceRepoOption, ) (repo.ITraceRepo, error) { - return &TraceCkRepoImpl{ - spansDao: spanDao, - annoDao: annoDao, - traceConfig: traceConfig, - spanRedisDao: spanRedisDao, - spanProducer: spanProducer, - }, nil + impl, err := newTraceRepoImpl(traceConfig, storageProvider, spanRedisDao, spanProducer, opts...) + if err != nil { + return nil, err + } + return impl, nil } func NewTraceMetricCKRepoImpl( - spanDao ck.ISpansDao, - annoDao ck.IAnnotationDao, traceConfig config.ITraceConfig, + storageProvider storage.IStorageProvider, + opts ...TraceRepoOption, ) (metric_repo.IMetricRepo, error) { - return &TraceCkRepoImpl{ - spansDao: spanDao, - annoDao: annoDao, - traceConfig: traceConfig, - }, nil + return newTraceRepoImpl(traceConfig, storageProvider, nil, nil, opts...) } -type TraceCkRepoImpl struct { - spansDao ck.ISpansDao - annoDao ck.IAnnotationDao - traceConfig config.ITraceConfig - spanRedisDao redis.ISpansRedisDao - spanProducer mq.ISpanProducer +func newTraceRepoImpl( + traceConfig config.ITraceConfig, + storageProvider storage.IStorageProvider, + spanRedisDao redis_dao.ISpansRedisDao, + spanProducer mq.ISpanProducer, + opts ...TraceRepoOption, +) (*TraceRepoImpl, error) { + impl := &TraceRepoImpl{ + traceConfig: traceConfig, + storageProvider: storageProvider, + spanDaos: make(map[string]dao.ISpansDao), + annoDaos: make(map[string]dao.IAnnotationDao), + spanRedisDao: spanRedisDao, + spanProducer: spanProducer, + } + for _, opt := range opts { + if opt != nil { + opt(impl) + } + } + return impl, nil } -func (t *TraceCkRepoImpl) GetPreSpanIDs(ctx context.Context, param *repo.GetPreSpanIDsParam) (preSpanIDs, responseIDs []string, err error) { +type TraceRepoImpl struct { + traceConfig config.ITraceConfig + storageProvider storage.IStorageProvider + spanDaos map[string]dao.ISpansDao + annoDaos map[string]dao.IAnnotationDao + spanRedisDao redis_dao.ISpansRedisDao + spanProducer mq.ISpanProducer +} + +func (t *TraceRepoImpl) GetPreSpanIDs(ctx context.Context, param *repo.GetPreSpanIDsParam) (preSpanIDs, responseIDs []string, err error) { return t.spanRedisDao.GetPreSpans(ctx, param.PreRespID) } @@ -74,14 +127,18 @@ type PageToken struct { SpanID string `json:"SpanID"` } -func (t *TraceCkRepoImpl) InsertSpans(ctx context.Context, param *repo.InsertTraceParam) error { +func (t *TraceRepoImpl) InsertSpans(ctx context.Context, param *repo.InsertTraceParam) error { + spanDao := t.spanDaos[ck.TraceStorageTypeCK] + if spanDao == nil { + return errorx.WrapByCode(errors.New("invalid storage"), obErrorx.CommercialCommonInvalidParamCodeCode) + } table, err := t.getSpanInsertTable(ctx, param.Tenant, param.TTL) if err != nil { return err } - if err := t.spansDao.Insert(ctx, &ck.InsertParam{ + if err := spanDao.Insert(ctx, &dao.InsertParam{ Table: table, - Spans: convertor.SpanListDO2PO(param.Spans, param.TTL), + Spans: converter.SpanListDO2PO(param.Spans, param.TTL), }); err != nil { logs.CtxError(ctx, "fail to insert spans, %v", err) return err @@ -90,7 +147,17 @@ func (t *TraceCkRepoImpl) InsertSpans(ctx context.Context, param *repo.InsertTra return nil } -func (t *TraceCkRepoImpl) ListSpans(ctx context.Context, req *repo.ListSpansParam) (*repo.ListSpansResult, error) { +func (t *TraceRepoImpl) ListSpans(ctx context.Context, req *repo.ListSpansParam) (*repo.ListSpansResult, error) { + spanStorage := t.storageProvider.GetTraceStorage(ctx, req.WorkSpaceID, req.Tenants) + spanDao := t.spanDaos[spanStorage.StorageName] + if spanDao == nil { + return nil, errorx.WrapByCode(errors.New("invalid storage"), obErrorx.CommercialCommonInvalidParamCodeCode) + } + annoDao := t.annoDaos[spanStorage.StorageName] + if annoDao == nil { + return nil, errorx.WrapByCode(errors.New("invalid storage"), obErrorx.CommercialCommonInvalidParamCodeCode) + } + pageToken, err := parsePageToken(req.PageToken) if err != nil { return nil, errorx.WrapByCode(err, obErrorx.CommercialCommonInvalidParamCodeCode, errorx.WithExtraMsg("invalid list spans request")) @@ -103,8 +170,8 @@ func (t *TraceCkRepoImpl) ListSpans(ctx context.Context, req *repo.ListSpansPara return nil, err } st := time.Now() - spans, err := t.spansDao.Get(ctx, &ck.QueryParam{ - QueryType: ck.QueryTypeListSpans, + spans, err := spanDao.Get(ctx, &dao.QueryParam{ + QueryType: dao.QueryTypeListSpans, Tables: tableCfg.SpanTables, AnnoTableMap: tableCfg.AnnoTableMap, StartTime: time_util.MillSec2MicroSec(req.StartAt), @@ -113,30 +180,32 @@ func (t *TraceCkRepoImpl) ListSpans(ctx context.Context, req *repo.ListSpansPara Limit: req.Limit + 1, OrderByStartTime: req.DescByStartTime, OmitColumns: req.OmitColumns, + Extra: spanStorage.StorageConfig, SelectColumns: req.SelectColumns, }) if err != nil { return nil, err } logs.CtxInfo(ctx, "list spans successfully, spans count %d, cost %v", len(spans), time.Since(st)) - spanDOList := convertor.SpanListPO2DO(spans) + spanDOList := converter.SpanListPO2DO(spans) if tableCfg.NeedQueryAnno && !req.NotQueryAnnotation { - spanIDs := lo.UniqMap(spans, func(item *model.ObservabilitySpan, _ int) string { + spanIDs := lo.UniqMap(spans, func(item *dao.Span, _ int) string { return item.SpanID }) st = time.Now() - annotations, err := t.annoDao.List(ctx, &ck.ListAnnotationsParam{ + annotations, err := annoDao.List(ctx, &dao.ListAnnotationsParam{ Tables: tableCfg.AnnoTables, SpanIDs: spanIDs, StartTime: time_util.MillSec2MicroSec(req.StartAt), EndTime: time_util.MillSec2MicroSec(req.EndAt), Limit: int32(min(len(spanIDs)*100, 10000)), + Extra: spanStorage.StorageConfig, }) logs.CtxInfo(ctx, "get annotations successfully, annotations count %d, cost %v", len(annotations), time.Since(st)) if err != nil { return nil, err } - annoDOList := convertor.AnnotationListPO2DO(annotations) + annoDOList := converter.AnnotationListPO2DO(annotations) spanDOList.SetAnnotations(annoDOList) } result := &repo.ListSpansResult{ @@ -159,7 +228,17 @@ func (t *TraceCkRepoImpl) ListSpans(ctx context.Context, req *repo.ListSpansPara return result, nil } -func (t *TraceCkRepoImpl) GetTrace(ctx context.Context, req *repo.GetTraceParam) (loop_span.SpanList, error) { +func (t *TraceRepoImpl) GetTrace(ctx context.Context, req *repo.GetTraceParam) (loop_span.SpanList, error) { + spanStorage := t.storageProvider.GetTraceStorage(ctx, req.WorkSpaceID, req.Tenants) + spanDao := t.spanDaos[spanStorage.StorageName] + if spanDao == nil { + return nil, errorx.WrapByCode(errors.New("invalid storage"), obErrorx.CommercialCommonInvalidParamCodeCode) + } + annoDao := t.annoDaos[spanStorage.StorageName] + if annoDao == nil { + return nil, errorx.WrapByCode(errors.New("invalid storage"), obErrorx.CommercialCommonInvalidParamCodeCode) + } + tableCfg, err := t.getQueryTenantTables(ctx, req.Tenants) if err != nil { return nil, err @@ -194,8 +273,8 @@ func (t *TraceCkRepoImpl) GetTrace(ctx context.Context, req *repo.GetTraceParam) SubFilter: req.Filters, }) st := time.Now() - spans, err := t.spansDao.Get(ctx, &ck.QueryParam{ - QueryType: ck.QueryTypeGetTrace, + spans, err := spanDao.Get(ctx, &dao.QueryParam{ + QueryType: dao.QueryTypeGetTrace, Tables: tableCfg.SpanTables, AnnoTableMap: tableCfg.AnnoTableMap, StartTime: time_util.MillSec2MicroSec(req.StartAt), @@ -204,36 +283,44 @@ func (t *TraceCkRepoImpl) GetTrace(ctx context.Context, req *repo.GetTraceParam) Limit: req.Limit, OmitColumns: req.OmitColumns, SelectColumns: req.SelectColumns, + Extra: spanStorage.StorageConfig, }) if err != nil { return nil, err } logs.CtxInfo(ctx, "get trace %s successfully, spans count %d, cost %v", req.TraceID, len(spans), time.Since(st)) - spanDOList := convertor.SpanListPO2DO(spans) + spanDOList := converter.SpanListPO2DO(spans) if tableCfg.NeedQueryAnno && !req.NotQueryAnnotation { - spanIDs := lo.UniqMap(spans, func(item *model.ObservabilitySpan, _ int) string { + spanIDs := lo.UniqMap(spans, func(item *dao.Span, _ int) string { return item.SpanID }) st = time.Now() - annotations, err := t.annoDao.List(ctx, &ck.ListAnnotationsParam{ + annotations, err := annoDao.List(ctx, &dao.ListAnnotationsParam{ Tables: tableCfg.AnnoTables, SpanIDs: spanIDs, StartTime: time_util.MillSec2MicroSec(req.StartAt), EndTime: time_util.MillSec2MicroSec(req.EndAt), Limit: int32(min(len(spanIDs)*100, 10000)), + Extra: spanStorage.StorageConfig, }) logs.CtxInfo(ctx, "get annotations successfully, annotations count %d, cost %v", len(annotations), time.Since(st)) if err != nil { return nil, err } - annoDOList := convertor.AnnotationListPO2DO(annotations) + annoDOList := converter.AnnotationListPO2DO(annotations) spanDOList.SetAnnotations(annoDOList.Uniq()) } return spanDOList.Uniq(), nil } -func (t *TraceCkRepoImpl) ListAnnotations(ctx context.Context, param *repo.ListAnnotationsParam) (loop_span.AnnotationList, error) { +func (t *TraceRepoImpl) ListAnnotations(ctx context.Context, param *repo.ListAnnotationsParam) (loop_span.AnnotationList, error) { + spanStorage := t.storageProvider.GetTraceStorage(ctx, param.WorkSpaceID, param.Tenants) + annoDao := t.annoDaos[spanStorage.StorageName] + if annoDao == nil { + return nil, errorx.WrapByCode(errors.New("invalid storage"), obErrorx.CommercialCommonInvalidParamCodeCode) + } + if param.SpanID == "" || param.TraceID == "" || param.WorkspaceId <= 0 { return nil, errorx.NewByCode(obErrorx.CommercialCommonInvalidParamCodeCode) } @@ -244,26 +331,33 @@ func (t *TraceCkRepoImpl) ListAnnotations(ctx context.Context, param *repo.ListA return loop_span.AnnotationList{}, nil } st := time.Now() - annotations, err := t.annoDao.List(ctx, &ck.ListAnnotationsParam{ + annotations, err := annoDao.List(ctx, &dao.ListAnnotationsParam{ Tables: tableCfg.AnnoTables, SpanIDs: []string{param.SpanID}, StartTime: time_util.MillSec2MicroSec(param.StartAt), EndTime: time_util.MillSec2MicroSec(param.EndAt), DescByUpdatedAt: param.DescByUpdatedAt, Limit: 100, + Extra: spanStorage.StorageConfig, }) if err != nil { return nil, err } logs.CtxInfo(ctx, "get annotations successfully, annotations count %d, cost %v", len(annotations), time.Since(st)) workspaceIDStr := strconv.FormatInt(param.WorkspaceId, 10) - annotations = lo.Filter(annotations, func(item *model.ObservabilityAnnotation, _ int) bool { + annotations = lo.Filter(annotations, func(item *dao.Annotation, _ int) bool { return item.TraceID == param.TraceID && item.SpaceID == workspaceIDStr }) - return convertor.AnnotationListPO2DO(annotations).Uniq(), nil + return converter.AnnotationListPO2DO(annotations).Uniq(), nil } -func (t *TraceCkRepoImpl) GetAnnotation(ctx context.Context, param *repo.GetAnnotationParam) (*loop_span.Annotation, error) { +func (t *TraceRepoImpl) GetAnnotation(ctx context.Context, param *repo.GetAnnotationParam) (*loop_span.Annotation, error) { + spanStorage := t.storageProvider.GetTraceStorage(ctx, param.WorkSpaceID, param.Tenants) + annoDao := t.annoDaos[spanStorage.StorageName] + if annoDao == nil { + return nil, errorx.WrapByCode(errors.New("invalid storage"), obErrorx.CommercialCommonInvalidParamCodeCode) + } + tableCfg, err := t.getQueryTenantTables(ctx, param.Tenants) if err != nil { return nil, err @@ -271,36 +365,44 @@ func (t *TraceCkRepoImpl) GetAnnotation(ctx context.Context, param *repo.GetAnno return nil, nil } st := time.Now() - annotation, err := t.annoDao.Get(ctx, &ck.GetAnnotationParam{ + annotation, err := annoDao.Get(ctx, &dao.GetAnnotationParam{ Tables: tableCfg.AnnoTables, ID: param.ID, StartTime: time_util.MillSec2MicroSec(param.StartAt), EndTime: time_util.MillSec2MicroSec(param.EndAt), Limit: 2, + Extra: spanStorage.StorageConfig, }) if err != nil { return nil, err } logs.CtxInfo(ctx, "get annotation successfully, cost %v", time.Since(st)) - return convertor.AnnotationPO2DO(annotation), nil + return converter.AnnotationPO2DO(annotation), nil } -func (t *TraceCkRepoImpl) InsertAnnotations(ctx context.Context, param *repo.InsertAnnotationParam) error { +func (t *TraceRepoImpl) InsertAnnotations(ctx context.Context, param *repo.InsertAnnotationParam) error { + spanStorage := t.storageProvider.GetTraceStorage(ctx, param.WorkSpaceID, []string{param.Tenant}) + annoDao := t.annoDaos[spanStorage.StorageName] + if annoDao == nil { + return errorx.WrapByCode(errors.New("invalid storage"), obErrorx.CommercialCommonInvalidParamCodeCode) + } + table, err := t.getAnnoInsertTable(ctx, param.Tenant, param.TTL) if err != nil { return err } - pos := make([]*model.ObservabilityAnnotation, 0, len(param.Span.Annotations)) + pos := make([]*dao.Annotation, 0, len(param.Span.Annotations)) for _, annotation := range param.Span.Annotations { - annotationPO, err := convertor.AnnotationDO2PO(annotation) + annotationPO, err := converter.AnnotationDO2PO(annotation) if err != nil { return err } pos = append(pos, annotationPO) } - err = t.annoDao.Insert(ctx, &ck.InsertAnnotationParam{ + err = annoDao.Insert(ctx, &dao.InsertAnnotationParam{ Table: table, Annotations: pos, + Extra: spanStorage.StorageConfig, }) if err != nil { return nil @@ -315,13 +417,19 @@ func (t *TraceCkRepoImpl) InsertAnnotations(ctx context.Context, param *repo.Ins }, annotationType) } -func (t *TraceCkRepoImpl) GetMetrics(ctx context.Context, param *metric_repo.GetMetricsParam) (*metric_repo.GetMetricsResult, error) { +func (t *TraceRepoImpl) GetMetrics(ctx context.Context, param *metric_repo.GetMetricsParam) (*metric_repo.GetMetricsResult, error) { + spanStorage := t.storageProvider.GetTraceStorage(ctx, param.WorkSpaceID, param.Tenants) + spanDao := t.spanDaos[spanStorage.StorageName] + if spanDao == nil { + return nil, errorx.WrapByCode(errors.New("invalid storage"), obErrorx.CommercialCommonInvalidParamCodeCode) + } + tableCfg, err := t.getQueryTenantTables(ctx, param.Tenants) if err != nil { return nil, err } st := time.Now() - metrics, err := t.spansDao.GetMetrics(ctx, &ck.GetMetricsParam{ + metrics, err := spanDao.GetMetrics(ctx, &dao.GetMetricsParam{ Tables: tableCfg.SpanTables, Aggregations: param.Aggregations, GroupBys: param.GroupBys, @@ -329,6 +437,7 @@ func (t *TraceCkRepoImpl) GetMetrics(ctx context.Context, param *metric_repo.Get StartAt: time_util.MillSec2MicroSec(param.StartAt), EndAt: time_util.MillSec2MicroSec(param.EndAt), Granularity: param.Granularity, + Extra: spanStorage.StorageConfig, }) if err != nil { return nil, err @@ -346,7 +455,7 @@ type queryTableCfg struct { NeedQueryAnno bool } -func (t *TraceCkRepoImpl) getQueryTenantTables(ctx context.Context, tenants []string) (*queryTableCfg, error) { +func (t *TraceRepoImpl) getQueryTenantTables(ctx context.Context, tenants []string) (*queryTableCfg, error) { tenantTableCfg, err := t.traceConfig.GetTenantConfig(ctx) if err != nil { logs.CtxError(ctx, "fail to get tenant table config, %v", err) @@ -383,7 +492,7 @@ func (t *TraceCkRepoImpl) getQueryTenantTables(ctx context.Context, tenants []st return ret, nil } -func (t *TraceCkRepoImpl) getSpanInsertTable(ctx context.Context, tenant string, ttl loop_span.TTL) (string, error) { +func (t *TraceRepoImpl) getSpanInsertTable(ctx context.Context, tenant string, ttl loop_span.TTL) (string, error) { tenantTableCfg, err := t.traceConfig.GetTenantConfig(ctx) if err != nil { logs.CtxError(ctx, "fail to get tenant config, %v", err) @@ -398,7 +507,7 @@ func (t *TraceCkRepoImpl) getSpanInsertTable(ctx context.Context, tenant string, return tableCfg.SpanTable, nil } -func (t *TraceCkRepoImpl) getAnnoInsertTable(ctx context.Context, tenant string, ttl loop_span.TTL) (string, error) { +func (t *TraceRepoImpl) getAnnoInsertTable(ctx context.Context, tenant string, ttl loop_span.TTL) (string, error) { tenantTableCfg, err := t.traceConfig.GetTenantConfig(ctx) if err != nil { logs.CtxError(ctx, "fail to get tenant config, %v", err) @@ -406,14 +515,14 @@ func (t *TraceCkRepoImpl) getAnnoInsertTable(ctx context.Context, tenant string, } tableCfg, ok := tenantTableCfg.TenantTables[tenant][ttl] if !ok { - return "", fmt.Errorf("no annotation table config found for tenant %s with ttl %s", tenant, ttl) + return "", nil } else if tableCfg.AnnoTable == "" { - return "", fmt.Errorf("no annotation table config found for tenant %s with ttl %s", tenant, ttl) + return "", nil } return tableCfg.AnnoTable, nil } -func (t *TraceCkRepoImpl) addPageTokenFilter(pageToken *PageToken, filter *loop_span.FilterFields) *loop_span.FilterFields { +func (t *TraceRepoImpl) addPageTokenFilter(pageToken *PageToken, filter *loop_span.FilterFields) *loop_span.FilterFields { timeStr := strconv.FormatInt(pageToken.StartTime, 10) filterFields := &loop_span.FilterFields{ QueryAndOr: ptr.Of(loop_span.QueryAndOrEnumOr), diff --git a/backend/modules/observability/infra/repo/trace_test.go b/backend/modules/observability/infra/repo/trace_test.go index 332518c46..5669632df 100644 --- a/backend/modules/observability/infra/repo/trace_test.go +++ b/backend/modules/observability/infra/repo/trace_test.go @@ -15,20 +15,41 @@ import ( confmocks "github.com/coze-dev/coze-loop/backend/modules/observability/domain/component/config/mocks" "github.com/coze-dev/coze-loop/backend/modules/observability/domain/component/mq" mqmock "github.com/coze-dev/coze-loop/backend/modules/observability/domain/component/mq/mocks" + "github.com/coze-dev/coze-loop/backend/modules/observability/domain/component/storage" metric_entity "github.com/coze-dev/coze-loop/backend/modules/observability/domain/metric/entity" metric_repo "github.com/coze-dev/coze-loop/backend/modules/observability/domain/metric/repo" "github.com/coze-dev/coze-loop/backend/modules/observability/domain/trace/entity/loop_span" - repo "github.com/coze-dev/coze-loop/backend/modules/observability/domain/trace/repo" - "github.com/coze-dev/coze-loop/backend/modules/observability/infra/repo/ck" - "github.com/coze-dev/coze-loop/backend/modules/observability/infra/repo/ck/gorm_gen/model" - ckmock "github.com/coze-dev/coze-loop/backend/modules/observability/infra/repo/ck/mocks" + "github.com/coze-dev/coze-loop/backend/modules/observability/domain/trace/repo" + "github.com/coze-dev/coze-loop/backend/modules/observability/infra/repo/dao" + daomock "github.com/coze-dev/coze-loop/backend/modules/observability/infra/repo/dao/mocks" + redis_dao "github.com/coze-dev/coze-loop/backend/modules/observability/infra/repo/redis" "github.com/coze-dev/coze-loop/backend/pkg/lang/ptr" - time_util "github.com/coze-dev/coze-loop/backend/pkg/time" ) -func TestTraceCkRepoImpl_InsertSpans(t *testing.T) { +type mockStorageProvider struct{} + +func (m *mockStorageProvider) GetTraceStorage(ctx context.Context, workSpaceID string, tenants []string) storage.Storage { + return storage.Storage{ + StorageName: "ck", + StorageConfig: map[string]string{}, + } +} + +func (m *mockStorageProvider) PrepareStorageForTask(ctx context.Context, workspaceID string, tenants []string) error { + return nil +} + +func (m *mockStorageProvider) GetSpanDao(tenant string) dao.ISpansDao { + return nil +} + +func (m *mockStorageProvider) GetAnnotationDao(tenant string) dao.IAnnotationDao { + return nil +} + +func TestTraceRepoImpl_InsertSpans(t *testing.T) { type fields struct { - spansDao ck.ISpansDao + spansDao dao.ISpansDao traceConfig config.ITraceConfig } type args struct { @@ -44,7 +65,7 @@ func TestTraceCkRepoImpl_InsertSpans(t *testing.T) { { name: "insert spans successfully", fieldsGetter: func(ctrl *gomock.Controller) fields { - spansDaoMock := ckmock.NewMockISpansDao(ctrl) + spansDaoMock := daomock.NewMockISpansDao(ctrl) spansDaoMock.EXPECT().Insert(gomock.Any(), gomock.Any()).Return(nil) traceConfigMock := confmocks.NewMockITraceConfig(ctrl) traceConfigMock.EXPECT().GetTenantConfig(gomock.Any()).Return(&config.TenantCfg{ @@ -84,7 +105,7 @@ func TestTraceCkRepoImpl_InsertSpans(t *testing.T) { { name: "insert spans failed due to dao error", fieldsGetter: func(ctrl *gomock.Controller) fields { - spansDaoMock := ckmock.NewMockISpansDao(ctrl) + spansDaoMock := daomock.NewMockISpansDao(ctrl) spansDaoMock.EXPECT().Insert(gomock.Any(), gomock.Any()).Return(assert.AnError) traceConfigMock := confmocks.NewMockITraceConfig(ctrl) traceConfigMock.EXPECT().GetTenantConfig(gomock.Any()).Return(&config.TenantCfg{ @@ -124,20 +145,24 @@ func TestTraceCkRepoImpl_InsertSpans(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() fields := tt.fieldsGetter(ctrl) - r := &TraceCkRepoImpl{ - spansDao: fields.spansDao, - traceConfig: fields.traceConfig, - } - err := r.InsertSpans(tt.args.ctx, tt.args.param) + r, err := NewTraceRepoImpl( + fields.traceConfig, + &mockStorageProvider{}, + nil, + nil, + WithTraceStorageSpanDao("ck", fields.spansDao), + ) + assert.NoError(t, err) + err = r.InsertSpans(tt.args.ctx, tt.args.param) assert.Equal(t, tt.wantErr, err != nil) }) } } -func TestTraceCkRepoImpl_ListSpans(t *testing.T) { +func TestTraceRepoImpl_ListSpans(t *testing.T) { type fields struct { - spansDao ck.ISpansDao - annoDao ck.IAnnotationDao + spansDao dao.ISpansDao + annoDao dao.IAnnotationDao traceConfig config.ITraceConfig } type args struct { @@ -154,8 +179,8 @@ func TestTraceCkRepoImpl_ListSpans(t *testing.T) { { name: "list spans successfully", fieldsGetter: func(ctrl *gomock.Controller) fields { - spansDaoMock := ckmock.NewMockISpansDao(ctrl) - spansDaoMock.EXPECT().Get(gomock.Any(), gomock.Any()).Return([]*model.ObservabilitySpan{ + spansDaoMock := daomock.NewMockISpansDao(ctrl) + spansDaoMock.EXPECT().Get(gomock.Any(), gomock.Any()).Return([]*dao.Span{ { TraceID: "123", SpanID: "123", @@ -191,7 +216,7 @@ func TestTraceCkRepoImpl_ListSpans(t *testing.T) { }, nil) return fields{ spansDao: spansDaoMock, - annoDao: ckmock.NewMockIAnnotationDao(ctrl), + annoDao: daomock.NewMockIAnnotationDao(ctrl), traceConfig: traceConfigMock, } }, @@ -232,8 +257,8 @@ func TestTraceCkRepoImpl_ListSpans(t *testing.T) { traceConfigMock := confmocks.NewMockITraceConfig(ctrl) traceConfigMock.EXPECT().GetTenantConfig(gomock.Any()).Return(nil, assert.AnError) return fields{ - spansDao: ckmock.NewMockISpansDao(ctrl), - annoDao: ckmock.NewMockIAnnotationDao(ctrl), + spansDao: daomock.NewMockISpansDao(ctrl), + annoDao: daomock.NewMockIAnnotationDao(ctrl), traceConfig: traceConfigMock, } }, @@ -249,14 +274,14 @@ func TestTraceCkRepoImpl_ListSpans(t *testing.T) { { name: "list spans with annotations successfully", fieldsGetter: func(ctrl *gomock.Controller) fields { - spansDaoMock := ckmock.NewMockISpansDao(ctrl) - spansDaoMock.EXPECT().Get(gomock.Any(), gomock.Any()).Return([]*model.ObservabilitySpan{ + spansDaoMock := daomock.NewMockISpansDao(ctrl) + spansDaoMock.EXPECT().Get(gomock.Any(), gomock.Any()).Return([]*dao.Span{ { SpanID: "span1", }, }, nil) - annoDaoMock := ckmock.NewMockIAnnotationDao(ctrl) - annoDaoMock.EXPECT().List(gomock.Any(), gomock.Any()).Return([]*model.ObservabilityAnnotation{ + annoDaoMock := daomock.NewMockIAnnotationDao(ctrl) + annoDaoMock.EXPECT().List(gomock.Any(), gomock.Any()).Return([]*dao.Annotation{ { ID: "anno1", SpanID: "span1", @@ -322,11 +347,14 @@ func TestTraceCkRepoImpl_ListSpans(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() fields := tt.fieldsGetter(ctrl) - r := &TraceCkRepoImpl{ - spansDao: fields.spansDao, - annoDao: fields.annoDao, - traceConfig: fields.traceConfig, - } + r, err := NewTraceRepoImpl( + fields.traceConfig, + &mockStorageProvider{}, + nil, + nil, + WithTraceStorageDaos("ck", fields.spansDao, fields.annoDao), + ) + assert.NoError(t, err) got, err := r.ListSpans(tt.args.ctx, tt.args.req) assert.Equal(t, tt.wantErr, err != nil) if tt.want != nil && got != nil { @@ -337,10 +365,10 @@ func TestTraceCkRepoImpl_ListSpans(t *testing.T) { } } -func TestTraceCkRepoImpl_GetTrace(t *testing.T) { +func TestTraceRepoImpl_GetTrace(t *testing.T) { type fields struct { - spansDao ck.ISpansDao - annoDao ck.IAnnotationDao + spansDao dao.ISpansDao + annoDao dao.IAnnotationDao traceConfig config.ITraceConfig } type args struct { @@ -357,8 +385,9 @@ func TestTraceCkRepoImpl_GetTrace(t *testing.T) { { name: "get trace successfully", fieldsGetter: func(ctrl *gomock.Controller) fields { - spansDaoMock := ckmock.NewMockISpansDao(ctrl) - spansDaoMock.EXPECT().Get(gomock.Any(), gomock.Any()).Return([]*model.ObservabilitySpan{ + spansDaoMock := daomock.NewMockISpansDao(ctrl) + // 期望的QueryParam应该包含TraceID过滤条件 + spansDaoMock.EXPECT().Get(gomock.Any(), gomock.Any()).Return([]*dao.Span{ { TraceID: "span1", SpanID: "span1", @@ -388,6 +417,7 @@ func TestTraceCkRepoImpl_GetTrace(t *testing.T) { }, nil) return fields{ spansDao: spansDaoMock, + annoDao: daomock.NewMockIAnnotationDao(ctrl), traceConfig: traceConfigMock, } }, @@ -428,14 +458,14 @@ func TestTraceCkRepoImpl_GetTrace(t *testing.T) { { name: "get trace with annotations successfully", fieldsGetter: func(ctrl *gomock.Controller) fields { - spansDaoMock := ckmock.NewMockISpansDao(ctrl) - spansDaoMock.EXPECT().Get(gomock.Any(), gomock.Any()).Return([]*model.ObservabilitySpan{ + spansDaoMock := daomock.NewMockISpansDao(ctrl) + spansDaoMock.EXPECT().Get(gomock.Any(), gomock.Any()).Return([]*dao.Span{ { SpanID: "span1", }, }, nil) - annoDaoMock := ckmock.NewMockIAnnotationDao(ctrl) - annoDaoMock.EXPECT().List(gomock.Any(), gomock.Any()).Return([]*model.ObservabilityAnnotation{ + annoDaoMock := daomock.NewMockIAnnotationDao(ctrl) + annoDaoMock.EXPECT().List(gomock.Any(), gomock.Any()).Return([]*dao.Annotation{ { ID: "anno1", SpanID: "span1", @@ -498,6 +528,8 @@ func TestTraceCkRepoImpl_GetTrace(t *testing.T) { traceConfigMock := confmocks.NewMockITraceConfig(ctrl) traceConfigMock.EXPECT().GetTenantConfig(gomock.Any()).Return(nil, assert.AnError) return fields{ + spansDao: daomock.NewMockISpansDao(ctrl), + annoDao: daomock.NewMockIAnnotationDao(ctrl), traceConfig: traceConfigMock, } }, @@ -513,8 +545,8 @@ func TestTraceCkRepoImpl_GetTrace(t *testing.T) { { name: "get trace with span successfully", fieldsGetter: func(ctrl *gomock.Controller) fields { - spansDaoMock := ckmock.NewMockISpansDao(ctrl) - spansDaoMock.EXPECT().Get(gomock.Any(), gomock.Any()).Return([]*model.ObservabilitySpan{ + spansDaoMock := daomock.NewMockISpansDao(ctrl) + spansDaoMock.EXPECT().Get(gomock.Any(), gomock.Any()).Return([]*dao.Span{ { TraceID: "span1", SpanID: "span1", @@ -536,6 +568,7 @@ func TestTraceCkRepoImpl_GetTrace(t *testing.T) { }, nil) return fields{ spansDao: spansDaoMock, + annoDao: daomock.NewMockIAnnotationDao(ctrl), traceConfig: traceConfigMock, } }, @@ -568,11 +601,14 @@ func TestTraceCkRepoImpl_GetTrace(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() fields := tt.fieldsGetter(ctrl) - r := &TraceCkRepoImpl{ - spansDao: fields.spansDao, - annoDao: fields.annoDao, - traceConfig: fields.traceConfig, - } + r, err := NewTraceRepoImpl( + fields.traceConfig, + &mockStorageProvider{}, + nil, + nil, + WithTraceStorageDaos("ck", fields.spansDao, fields.annoDao), + ) + assert.NoError(t, err) got, err := r.GetTrace(tt.args.ctx, tt.args.req) assert.Equal(t, err != nil, tt.wantErr) assert.Equal(t, tt.want, got) @@ -580,12 +616,12 @@ func TestTraceCkRepoImpl_GetTrace(t *testing.T) { } } -func TestTraceCkRepoImpl_GetMetrics(t *testing.T) { +func TestTraceRepoImpl_GetMetrics(t *testing.T) { t.Run("get metrics successfully", func(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - spansDaoMock := ckmock.NewMockISpansDao(ctrl) + spansDaoMock := daomock.NewMockISpansDao(ctrl) traceConfigMock := confmocks.NewMockITraceConfig(ctrl) aggregations := []*metric_entity.Dimension{ @@ -615,21 +651,12 @@ func TestTraceCkRepoImpl_GetMetrics(t *testing.T) { }, }, } - expectedParam := &ck.GetMetricsParam{ - Tables: []string{"spans"}, - Aggregations: aggregations, - GroupBys: groupBys, - Filters: filters, - StartAt: time_util.MillSec2MicroSec(1000), - EndAt: time_util.MillSec2MicroSec(2000), - Granularity: metric_entity.MetricGranularity1Min, - } metricsData := []map[string]any{ { "count": 1, }, } - spansDaoMock.EXPECT().GetMetrics(gomock.Any(), gomock.Eq(expectedParam)).Return(metricsData, nil) + spansDaoMock.EXPECT().GetMetrics(gomock.Any(), gomock.Any()).Return(metricsData, nil) traceConfigMock.EXPECT().GetTenantConfig(gomock.Any()).Return(&config.TenantCfg{ TenantTables: map[string]map[loop_span.TTL]config.TableCfg{ "tenant": { @@ -640,10 +667,12 @@ func TestTraceCkRepoImpl_GetMetrics(t *testing.T) { }, }, nil) - repoImpl := &TraceCkRepoImpl{ - spansDao: spansDaoMock, - traceConfig: traceConfigMock, - } + repoImpl, err := NewTraceMetricCKRepoImpl( + traceConfigMock, + &mockStorageProvider{}, + WithTraceStorageSpanDao("ck", spansDaoMock), + ) + assert.NoError(t, err) result, err := repoImpl.GetMetrics(context.Background(), &metric_repo.GetMetricsParam{ Tenants: []string{"tenant"}, Aggregations: aggregations, @@ -664,10 +693,13 @@ func TestTraceCkRepoImpl_GetMetrics(t *testing.T) { traceConfigMock := confmocks.NewMockITraceConfig(ctrl) traceConfigMock.EXPECT().GetTenantConfig(gomock.Any()).Return(nil, assert.AnError) - repoImpl := &TraceCkRepoImpl{ - traceConfig: traceConfigMock, - spansDao: ckmock.NewMockISpansDao(ctrl), - } + spansDaoMock := daomock.NewMockISpansDao(ctrl) + repoImpl, err := NewTraceMetricCKRepoImpl( + traceConfigMock, + &mockStorageProvider{}, + WithTraceStorageSpanDao("ck", spansDaoMock), + ) + assert.NoError(t, err) result, err := repoImpl.GetMetrics(context.Background(), &metric_repo.GetMetricsParam{ Tenants: []string{"tenant"}, }) @@ -679,7 +711,7 @@ func TestTraceCkRepoImpl_GetMetrics(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - spansDaoMock := ckmock.NewMockISpansDao(ctrl) + spansDaoMock := daomock.NewMockISpansDao(ctrl) traceConfigMock := confmocks.NewMockITraceConfig(ctrl) traceConfigMock.EXPECT().GetTenantConfig(gomock.Any()).Return(&config.TenantCfg{ @@ -693,10 +725,12 @@ func TestTraceCkRepoImpl_GetMetrics(t *testing.T) { }, nil) spansDaoMock.EXPECT().GetMetrics(gomock.Any(), gomock.Any()).Return(nil, assert.AnError) - repoImpl := &TraceCkRepoImpl{ - spansDao: spansDaoMock, - traceConfig: traceConfigMock, - } + repoImpl, err := NewTraceMetricCKRepoImpl( + traceConfigMock, + &mockStorageProvider{}, + WithTraceStorageSpanDao("ck", spansDaoMock), + ) + assert.NoError(t, err) result, err := repoImpl.GetMetrics(context.Background(), &metric_repo.GetMetricsParam{ Tenants: []string{"tenant"}, }) @@ -705,9 +739,9 @@ func TestTraceCkRepoImpl_GetMetrics(t *testing.T) { }) } -func TestTraceCkRepoImpl_InsertAnnotation(t *testing.T) { +func TestTraceRepoImpl_InsertAnnotation(t *testing.T) { type fields struct { - annoDao ck.IAnnotationDao + annoDao dao.IAnnotationDao traceConfig config.ITraceConfig spanProducer mq.ISpanProducer } @@ -724,7 +758,7 @@ func TestTraceCkRepoImpl_InsertAnnotation(t *testing.T) { { name: "insert annotation successfully", fieldsGetter: func(ctrl *gomock.Controller) fields { - annoDaoMock := ckmock.NewMockIAnnotationDao(ctrl) + annoDaoMock := daomock.NewMockIAnnotationDao(ctrl) annoDaoMock.EXPECT().Insert(gomock.Any(), gomock.Any()).Return(nil) traceConfigMock := confmocks.NewMockITraceConfig(ctrl) traceConfigMock.EXPECT().GetTenantConfig(gomock.Any()).Return(&config.TenantCfg{ @@ -768,6 +802,7 @@ func TestTraceCkRepoImpl_InsertAnnotation(t *testing.T) { traceConfigMock.EXPECT().GetTenantConfig(gomock.Any()).Return(nil, assert.AnError) spanProducerMock := mqmock.NewMockISpanProducer(ctrl) return fields{ + annoDao: daomock.NewMockIAnnotationDao(ctrl), traceConfig: traceConfigMock, spanProducer: spanProducerMock, } @@ -795,21 +830,26 @@ func TestTraceCkRepoImpl_InsertAnnotation(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() fields := tt.fieldsGetter(ctrl) - r := &TraceCkRepoImpl{ - annoDao: fields.annoDao, - traceConfig: fields.traceConfig, - spanProducer: fields.spanProducer, - } - err := r.InsertAnnotations(tt.args.ctx, tt.args.param) + r, err := NewTraceRepoImpl( + fields.traceConfig, + &mockStorageProvider{}, + nil, + fields.spanProducer, + WithTraceStorageAnnotationDao("ck", fields.annoDao), + ) + assert.NoError(t, err) + err = r.InsertAnnotations(tt.args.ctx, tt.args.param) assert.Equal(t, tt.wantErr, err != nil) }) } } -func TestTraceCkRepoImpl_GetAnnotation(t *testing.T) { +func TestTraceRepoImpl_GetAnnotation(t *testing.T) { type fields struct { - annoDao ck.IAnnotationDao - traceConfig config.ITraceConfig + annoDao dao.IAnnotationDao + traceConfig config.ITraceConfig + spanRedisDao redis_dao.ISpansRedisDao + spanProducer mq.ISpanProducer } type args struct { ctx context.Context @@ -825,8 +865,8 @@ func TestTraceCkRepoImpl_GetAnnotation(t *testing.T) { { name: "get annotation successfully", fieldsGetter: func(ctrl *gomock.Controller) fields { - annoDaoMock := ckmock.NewMockIAnnotationDao(ctrl) - annoDaoMock.EXPECT().Get(gomock.Any(), gomock.Any()).Return(&model.ObservabilityAnnotation{ + annoDaoMock := daomock.NewMockIAnnotationDao(ctrl) + annoDaoMock.EXPECT().Get(gomock.Any(), gomock.Any()).Return(&dao.Annotation{ ID: "anno1", }, nil) traceConfigMock := confmocks.NewMockITraceConfig(ctrl) @@ -840,8 +880,10 @@ func TestTraceCkRepoImpl_GetAnnotation(t *testing.T) { }, }, nil) return fields{ - annoDao: annoDaoMock, - traceConfig: traceConfigMock, + annoDao: annoDaoMock, + traceConfig: traceConfigMock, + spanRedisDao: nil, + spanProducer: nil, } }, args: args{ @@ -864,7 +906,10 @@ func TestTraceCkRepoImpl_GetAnnotation(t *testing.T) { traceConfigMock := confmocks.NewMockITraceConfig(ctrl) traceConfigMock.EXPECT().GetTenantConfig(gomock.Any()).Return(nil, assert.AnError) return fields{ - traceConfig: traceConfigMock, + annoDao: daomock.NewMockIAnnotationDao(ctrl), + traceConfig: traceConfigMock, + spanRedisDao: nil, + spanProducer: nil, } }, args: args{ @@ -882,10 +927,14 @@ func TestTraceCkRepoImpl_GetAnnotation(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() fields := tt.fieldsGetter(ctrl) - r := &TraceCkRepoImpl{ - annoDao: fields.annoDao, - traceConfig: fields.traceConfig, - } + r, err := NewTraceRepoImpl( + fields.traceConfig, + &mockStorageProvider{}, + fields.spanRedisDao, + fields.spanProducer, + WithTraceStorageAnnotationDao("ck", fields.annoDao), + ) + assert.NoError(t, err) got, err := r.GetAnnotation(tt.args.ctx, tt.args.param) assert.Equal(t, tt.wantErr, err != nil) assert.Equal(t, tt.want, got) @@ -893,10 +942,12 @@ func TestTraceCkRepoImpl_GetAnnotation(t *testing.T) { } } -func TestTraceCkRepoImpl_ListAnnotations(t *testing.T) { +func TestTraceRepoImpl_ListAnnotations(t *testing.T) { type fields struct { - annoDao ck.IAnnotationDao - traceConfig config.ITraceConfig + annoDao dao.IAnnotationDao + traceConfig config.ITraceConfig + spanRedisDao redis_dao.ISpansRedisDao + spanProducer mq.ISpanProducer } type args struct { ctx context.Context @@ -912,8 +963,8 @@ func TestTraceCkRepoImpl_ListAnnotations(t *testing.T) { { name: "list annotations successfully", fieldsGetter: func(ctrl *gomock.Controller) fields { - annoDaoMock := ckmock.NewMockIAnnotationDao(ctrl) - annoDaoMock.EXPECT().List(gomock.Any(), gomock.Any()).Return([]*model.ObservabilityAnnotation{ + annoDaoMock := daomock.NewMockIAnnotationDao(ctrl) + annoDaoMock.EXPECT().List(gomock.Any(), gomock.Any()).Return([]*dao.Annotation{ { ID: "anno1", TraceID: "trace1", @@ -931,8 +982,10 @@ func TestTraceCkRepoImpl_ListAnnotations(t *testing.T) { }, }, nil) return fields{ - annoDao: annoDaoMock, - traceConfig: traceConfigMock, + annoDao: annoDaoMock, + traceConfig: traceConfigMock, + spanRedisDao: nil, + spanProducer: nil, } }, args: args{ @@ -972,10 +1025,14 @@ func TestTraceCkRepoImpl_ListAnnotations(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() fields := tt.fieldsGetter(ctrl) - r := &TraceCkRepoImpl{ - annoDao: fields.annoDao, - traceConfig: fields.traceConfig, - } + r, err := NewTraceRepoImpl( + fields.traceConfig, + &mockStorageProvider{}, + fields.spanRedisDao, + fields.spanProducer, + WithTraceStorageAnnotationDao("ck", fields.annoDao), + ) + assert.NoError(t, err) got, err := r.ListAnnotations(tt.args.ctx, tt.args.param) assert.Equal(t, tt.wantErr, err != nil) assert.Equal(t, tt.want, got) diff --git a/backend/modules/observability/infra/storage/storage.go b/backend/modules/observability/infra/storage/storage.go new file mode 100644 index 000000000..bd69e32e9 --- /dev/null +++ b/backend/modules/observability/infra/storage/storage.go @@ -0,0 +1,25 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 +package storage + +import ( + "context" + "github.com/coze-dev/coze-loop/backend/modules/observability/domain/component/storage" +) + +type TraceStorageProviderImpl struct { +} + +func NewTraceStorageProvider() storage.IStorageProvider { + return &TraceStorageProviderImpl{} +} + +func (r *TraceStorageProviderImpl) GetTraceStorage(ctx context.Context, workspaceID string, tenants []string) storage.Storage { + return storage.Storage{ + StorageName: "ck", + } +} + +func (r *TraceStorageProviderImpl) PrepareStorageForTask(ctx context.Context, workspaceID string, tenants []string) error { + return nil +} diff --git a/backend/modules/observability/lib/go.mod b/backend/modules/observability/lib/go.mod new file mode 100644 index 000000000..c74c2c422 --- /dev/null +++ b/backend/modules/observability/lib/go.mod @@ -0,0 +1,33 @@ +module github.com/coze-dev/coze-loop/backend/modules/observability/lib + +go 1.24.6 + +require ( + github.com/bytedance/gg v1.1.0 + github.com/bytedance/sonic v1.14.2 + github.com/coze-dev/cozeloop-go/spec v0.1.6 + github.com/stretchr/testify v1.11.1 + go.opentelemetry.io/otel v1.38.0 + go.opentelemetry.io/proto/otlp v1.9.0 +) + +require ( + github.com/bytedance/gopkg v0.1.3 // indirect + github.com/bytedance/sonic/loader v0.4.0 // indirect + github.com/cloudwego/base64x v0.1.6 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 // indirect + github.com/klauspost/cpuid/v2 v2.2.10 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/rogpeppe/go-internal v1.9.0 // indirect + github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + golang.org/x/arch v0.15.0 // indirect + golang.org/x/net v0.43.0 // indirect + golang.org/x/sys v0.35.0 // indirect + golang.org/x/text v0.28.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 // indirect + google.golang.org/grpc v1.75.1 // indirect + google.golang.org/protobuf v1.36.10 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/backend/modules/observability/lib/go.sum b/backend/modules/observability/lib/go.sum new file mode 100644 index 000000000..7ae13f975 --- /dev/null +++ b/backend/modules/observability/lib/go.sum @@ -0,0 +1,89 @@ +github.com/bytedance/gg v1.1.0 h1:FSKRxOZeN30w7h6snEbHxzgVMUV7+Xu4gc/Lz1cmBFw= +github.com/bytedance/gg v1.1.0/go.mod h1:MeGhXyy5K20hNAU9GkMM51sXdm/lsqdU0CxwIiGvZpo= +github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M= +github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM= +github.com/bytedance/sonic v1.14.2 h1:k1twIoe97C1DtYUo+fZQy865IuHia4PR5RPiuGPPIIE= +github.com/bytedance/sonic v1.14.2/go.mod h1:T80iDELeHiHKSc0C9tubFygiuXoGzrkjKzX2quAx980= +github.com/bytedance/sonic/loader v0.4.0 h1:olZ7lEqcxtZygCK9EKYKADnpQoYkRQxaeY2NYzevs+o= +github.com/bytedance/sonic/loader v0.4.0/go.mod h1:AR4NYCk5DdzZizZ5djGqQ92eEhCCcdf5x77udYiSJRo= +github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M= +github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU= +github.com/coze-dev/cozeloop-go/spec v0.1.6 h1:lStq3CfvTwBn0y281X4KnhW7mtf9g/XIcccGEDSWTD0= +github.com/coze-dev/cozeloop-go/spec v0.1.6/go.mod h1:/f3BrWehffwXIpd4b5rYIqktLd/v5dlLBw0h9F/LQIU= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 h1:8Tjv8EJ+pM1xP8mK6egEbD1OgnVTyacbefKhmbLhIhU= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2/go.mod h1:pkJQ2tZHJ0aFOVEEot6oZmaVEZcRme73eIFmhiVuRWs= +github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= +github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= +github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8= +go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM= +go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA= +go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI= +go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI= +go.opentelemetry.io/otel/sdk v1.37.0/go.mod h1:VredYzxUvuo2q3WRcDnKDjbdvmO0sCzOvVAiY+yUkAg= +go.opentelemetry.io/otel/sdk/metric v1.37.0 h1:90lI228XrB9jCMuSdA0673aubgRobVZFhbjxHHspCPc= +go.opentelemetry.io/otel/sdk/metric v1.37.0/go.mod h1:cNen4ZWfiD37l5NhS+Keb5RXVWZWpRE+9WyVCpbo5ps= +go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE= +go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs= +go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A= +go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4= +golang.org/x/arch v0.15.0 h1:QtOrQd0bTUnhNVNndMpLHNWrDmYzZ2KDqSrEymqInZw= +golang.org/x/arch v0.15.0/go.mod h1:JmwW7aLIoRUKgaTzhkiEFxvcEiQGyOg9BMonBJUS7EE= +golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= +golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= +golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= +golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= +golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 h1:BIRfGDEjiHRrk0QKZe3Xv2ieMhtgRGeLcZQ0mIVn4EY= +google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5/go.mod h1:j3QtIyytwqGr1JUDtYXwtMXWPKsEa5LtzIFN1Wn5WvE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 h1:eaY8u2EuxbRv7c3NiGK0/NedzVsCcV6hDuU5qPX5EGE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5/go.mod h1:M4/wBTSeyLxupu3W3tJtOgB14jILAS/XWPSSa3TAlJc= +google.golang.org/grpc v1.75.1 h1:/ODCNEuf9VghjgO3rqLcfg8fiOP0nSluljWFlDxELLI= +google.golang.org/grpc v1.75.1/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ= +google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= +google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/backend/modules/observability/domain/trace/entity/otel/consts.go b/backend/modules/observability/lib/otel/consts.go similarity index 87% rename from backend/modules/observability/domain/trace/entity/otel/consts.go rename to backend/modules/observability/lib/otel/consts.go index 022b43f0b..7b09f061a 100644 --- a/backend/modules/observability/domain/trace/entity/otel/consts.go +++ b/backend/modules/observability/lib/otel/consts.go @@ -22,6 +22,9 @@ const ( otelAttributeOutput = "cozeloop.output" otelAttributeLogID = "cozeloop.logid" + apmInput = "gen_ai.input" + apmOutput = "gen_ai.output" + // model otelTraceLoopAttributeModelSpanType = "gen_ai.request.type" // traceloop span type otelAttributeModelTimeToFirstToken = "cozeloop.time_to_first_token" @@ -101,6 +104,7 @@ const ( tagKeyUserID = "user_id" tagKeyMessageID = "message_id" tagKeyStartTimeFirstResp = "start_time_first_resp" + tagKeyCallOptions = "call_options" ) var otelModelSpanTypeMap = map[string]string{ @@ -129,3 +133,27 @@ const ( dataTypeBool = "bool" dataTypeArrayString = "array_string" ) + +var LoopStringTags = []string{ + tagKeyThreadID, + tagKeyUserID, + tagKeyMessageID, + tracespec.Error, + tracespec.PromptProvider, + tracespec.ModelName, + tagKeyCallOptions, + tracespec.PromptKey, + tracespec.PromptVersion, + tracespec.PromptProvider, +} + +var LoopLongTags = []string{ + tracespec.LatencyFirstResp, + tracespec.InputTokens, + tracespec.OutputTokens, + tracespec.Tokens, +} + +var LoopBoolTags = []string{ + tracespec.Stream, +} diff --git a/backend/modules/observability/domain/trace/entity/otel/open_inference/openinference.go b/backend/modules/observability/lib/otel/open_inference/openinference.go similarity index 100% rename from backend/modules/observability/domain/trace/entity/otel/open_inference/openinference.go rename to backend/modules/observability/lib/otel/open_inference/openinference.go diff --git a/backend/modules/observability/domain/trace/entity/otel/open_inference/openinference_test.go b/backend/modules/observability/lib/otel/open_inference/openinference_test.go similarity index 100% rename from backend/modules/observability/domain/trace/entity/otel/open_inference/openinference_test.go rename to backend/modules/observability/lib/otel/open_inference/openinference_test.go diff --git a/backend/modules/observability/domain/trace/entity/otel/otel_convert.go b/backend/modules/observability/lib/otel/otel_convert.go similarity index 81% rename from backend/modules/observability/domain/trace/entity/otel/otel_convert.go rename to backend/modules/observability/lib/otel/otel_convert.go index 49f9d6b58..03ee902be 100644 --- a/backend/modules/observability/domain/trace/entity/otel/otel_convert.go +++ b/backend/modules/observability/lib/otel/otel_convert.go @@ -11,84 +11,81 @@ import ( "strings" "github.com/bytedance/gg/gptr" - - "github.com/coze-dev/coze-loop/backend/modules/observability/domain/trace/entity/loop_span" - "github.com/coze-dev/coze-loop/backend/modules/observability/domain/trace/entity/otel/open_inference" - "github.com/coze-dev/coze-loop/backend/pkg/logs" + "github.com/bytedance/sonic" + "github.com/coze-dev/coze-loop/backend/modules/observability/lib/otel/open_inference" + "github.com/coze-dev/cozeloop-go/spec/tracespec" semconv1_26_0 "go.opentelemetry.io/otel/semconv/v1.26.0" semconv1_27_0 "go.opentelemetry.io/otel/semconv/v1.27.0" semconv1_32_0 "go.opentelemetry.io/otel/semconv/v1.32.0" - - "github.com/bytedance/sonic" - "github.com/coze-dev/cozeloop-go/spec/tracespec" ) -// Field configuration, supports configuring data sources and export methods for fields, currently supports attribute, event, is_tag, data_type +// FieldConfMap Field configuration, supports configuring data sources and export methods for fields, currently supports attribute, event, is_tag, data_type // Among them, attributes and events support configuring multiple, while tags and datatypes only support configuring one. // Other types of configurations need to be manually processed in the code. var ( - fieldConfMap = map[string]fieldConf{ + FieldConfMap = map[string]FieldConf{ // common "span_type": { - attributeKey: []string{ + AttributeKey: []string{ otelAttributeSpanType, otelTraceLoopAttributeModelSpanType, string(semconv1_32_0.GenAIOperationNameKey), openInferenceAttributeSpanKind, }, - isTag: false, - dataType: dataTypeString, + IsTag: false, + DataType: dataTypeString, }, tagKeyThreadID: { - attributeKey: []string{string(semconv1_26_0.SessionIDKey)}, - isTag: true, - dataType: dataTypeString, + AttributeKey: []string{string(semconv1_26_0.SessionIDKey)}, + IsTag: true, + DataType: dataTypeString, }, tagKeyLogID: { - attributeKey: []string{otelAttributeLogID}, - isTag: true, - dataType: dataTypeString, + AttributeKey: []string{otelAttributeLogID}, + IsTag: true, + DataType: dataTypeString, }, tagKeyUserID: { - attributeKey: []string{string(semconv1_32_0.UserIDKey)}, - isTag: true, - dataType: dataTypeString, + AttributeKey: []string{string(semconv1_32_0.UserIDKey)}, + IsTag: true, + DataType: dataTypeString, }, tagKeyMessageID: { - attributeKey: []string{string(semconv1_32_0.MessagingMessageIDKey)}, - isTag: true, - dataType: dataTypeString, + AttributeKey: []string{string(semconv1_32_0.MessagingMessageIDKey)}, + IsTag: true, + DataType: dataTypeString, }, tracespec.Error: { - attributeKeyPrefix: []string{ + AttributeKeyPrefix: []string{ otelAttributeErrorPrefix, openInferenceAttributeException, }, - eventName: []string{semconv1_32_0.ExceptionEventName}, - isTag: true, - dataType: dataTypeString, + EventName: []string{semconv1_32_0.ExceptionEventName}, + IsTag: true, + DataType: dataTypeString, }, // model tracespec.ModelProvider: { - attributeKey: []string{string(semconv1_32_0.GenAISystemKey)}, - isTag: true, - dataType: dataTypeString, + AttributeKey: []string{string(semconv1_32_0.GenAISystemKey)}, + IsTag: true, + DataType: dataTypeString, }, tracespec.Input: { - attributeKey: []string{ + AttributeKey: []string{ openInferenceAttributeInput, springAIAttributeToolInput, otelAttributeInput, + apmInput, }, - attributeKeyPrefix: []string{ + AttributeKeyPrefix: []string{ openInferenceAttributeModelInputMessages, openInferenceAttributeToolInput, string(semconv1_27_0.GenAIPromptKey), }, - eventName: []string{otelEventModelSystemMessage, otelEventModelUserMessage, otelEventModelToolMessage, otelEventModelAssistantMessage, otelSpringAIEventModelPrompt}, - dataType: dataTypeString, + EventName: []string{otelEventModelSystemMessage, otelEventModelUserMessage, otelEventModelToolMessage, otelEventModelAssistantMessage, otelSpringAIEventModelPrompt}, + DataType: dataTypeString, eventHighLevelKey: []highLevelKeyRuleConf{ { key: "messages", @@ -103,17 +100,18 @@ var ( }, }, tracespec.Output: { - attributeKey: []string{ + AttributeKey: []string{ openInferenceAttributeOutput, springAIAttributeToolOutput, otelAttributeOutput, + apmOutput, }, - attributeKeyPrefix: []string{ + AttributeKeyPrefix: []string{ openInferenceAttributeModelOutputMessages, string(semconv1_27_0.GenAICompletionKey), }, - eventName: []string{otelEventModelChoice, otelSpringAIEventModelCompletion}, - dataType: dataTypeString, + EventName: []string{otelEventModelChoice, otelSpringAIEventModelCompletion}, + DataType: dataTypeString, eventHighLevelKey: []highLevelKeyRuleConf{ { key: "choices", @@ -132,103 +130,103 @@ var ( }, }, tagKeyStartTimeFirstResp: { - attributeKey: []string{otelAttributeModelTimeToFirstToken}, - isTag: true, - dataType: dataTypeInt64, + AttributeKey: []string{otelAttributeModelTimeToFirstToken}, + IsTag: true, + DataType: dataTypeInt64, }, tracespec.Stream: { - attributeKey: []string{otelAttributeModelStream}, - isTag: true, - dataType: dataTypeBool, + AttributeKey: []string{otelAttributeModelStream}, + IsTag: true, + DataType: dataTypeBool, }, tracespec.ModelName: { - attributeKey: []string{ + AttributeKey: []string{ string(semconv1_32_0.GenAIRequestModelKey), string(semconv1_27_0.GenAIResponseModelKey), openInferenceAttributeModelName, }, - isTag: true, - dataType: dataTypeString, + IsTag: true, + DataType: dataTypeString, }, "temperature": { - attributeKey: []string{string(semconv1_32_0.GenAIRequestTemperatureKey)}, - isTag: true, - dataType: dataTypeFloat64, + AttributeKey: []string{string(semconv1_32_0.GenAIRequestTemperatureKey)}, + IsTag: true, + DataType: dataTypeFloat64, }, "top_p": { - attributeKey: []string{string(semconv1_32_0.GenAIRequestTopPKey)}, - isTag: true, - dataType: dataTypeFloat64, + AttributeKey: []string{string(semconv1_32_0.GenAIRequestTopPKey)}, + IsTag: true, + DataType: dataTypeFloat64, }, "top_k": { - attributeKey: []string{string(semconv1_32_0.GenAIRequestTopKKey)}, - isTag: true, - dataType: dataTypeInt64, + AttributeKey: []string{string(semconv1_32_0.GenAIRequestTopKKey)}, + IsTag: true, + DataType: dataTypeInt64, }, "max_tokens": { - attributeKey: []string{string(semconv1_32_0.GenAIRequestMaxTokensKey)}, - isTag: true, - dataType: dataTypeInt64, + AttributeKey: []string{string(semconv1_32_0.GenAIRequestMaxTokensKey)}, + IsTag: true, + DataType: dataTypeInt64, }, "frequency_penalty": { - attributeKey: []string{string(semconv1_32_0.GenAIRequestFrequencyPenaltyKey)}, - isTag: true, - dataType: dataTypeFloat64, + AttributeKey: []string{string(semconv1_32_0.GenAIRequestFrequencyPenaltyKey)}, + IsTag: true, + DataType: dataTypeFloat64, }, "presence_penalty": { - attributeKey: []string{string(semconv1_32_0.GenAIRequestPresencePenaltyKey)}, - isTag: true, - dataType: dataTypeFloat64, + AttributeKey: []string{string(semconv1_32_0.GenAIRequestPresencePenaltyKey)}, + IsTag: true, + DataType: dataTypeFloat64, }, "stop_sequences": { - attributeKey: []string{string(semconv1_32_0.GenAIRequestStopSequencesKey)}, - isTag: true, - dataType: dataTypeArrayString, + AttributeKey: []string{string(semconv1_32_0.GenAIRequestStopSequencesKey)}, + IsTag: true, + DataType: dataTypeArrayString, }, tracespec.InputTokens: { - attributeKey: []string{ + AttributeKey: []string{ string(semconv1_32_0.GenAIUsageInputTokensKey), string(semconv1_26_0.GenAiUsagePromptTokensKey), openInferenceAttributeModelInputTokens, }, - isTag: true, - dataType: dataTypeInt64, + IsTag: true, + DataType: dataTypeInt64, }, tracespec.OutputTokens: { - attributeKey: []string{ + AttributeKey: []string{ string(semconv1_32_0.GenAIUsageOutputTokensKey), string(semconv1_26_0.GenAiUsageCompletionTokensKey), openInferenceAttributeModelOutputTokens, }, - isTag: true, - dataType: dataTypeInt64, + IsTag: true, + DataType: dataTypeInt64, }, // prompt tracespec.PromptKey: { - attributeKey: []string{otelAttributePromptKey}, - isTag: true, - dataType: dataTypeString, + AttributeKey: []string{otelAttributePromptKey}, + IsTag: true, + DataType: dataTypeString, }, tracespec.PromptVersion: { - attributeKey: []string{otelAttributePromptVersion}, - isTag: true, - dataType: dataTypeString, + AttributeKey: []string{otelAttributePromptVersion}, + IsTag: true, + DataType: dataTypeString, }, tracespec.PromptProvider: { - attributeKey: []string{otelAttributePromptProvider}, - isTag: true, - dataType: dataTypeString, + AttributeKey: []string{otelAttributePromptProvider}, + IsTag: true, + DataType: dataTypeString, }, } ) -type fieldConf struct { - attributeKey []string - attributeKeyPrefix []string - eventName []string - isTag bool - dataType string +type FieldConf struct { + AttributeKey []string + AttributeKeyPrefix []string + EventName []string + IsTag bool + DataType string eventHighLevelKey []highLevelKeyRuleConf // config from inner to outer, such as choices.message.xxx, config is ["message", "choices"] attributeHighLevelKey []highLevelKeyRuleConf // config from inner to outer, such as choices.message.xxx, config is ["message", "choices"] } @@ -253,18 +251,18 @@ var ( func init() { registeredAttributeMap = make(map[string]bool) registeredAttributePrefixMap = make(map[string]bool) - for _, fieldConf := range fieldConfMap { - for _, attribute := range fieldConf.attributeKey { + for _, fieldConf := range FieldConfMap { + for _, attribute := range fieldConf.AttributeKey { registeredAttributeMap[attribute] = true } - for _, attributePrefix := range fieldConf.attributeKeyPrefix { + for _, attributePrefix := range fieldConf.AttributeKeyPrefix { registeredAttributePrefixMap[attributePrefix] = true } } } -func OtelSpansConvertToSendSpans(ctx context.Context, spaceID string, spans []*ResourceScopeSpan) loop_span.SpanList { - result := make(loop_span.SpanList, 0) +func OtelSpansConvertToSendSpans(ctx context.Context, spaceID string, spans []*ResourceScopeSpan) []*LoopSpan { + result := make([]*LoopSpan, 0) for i := range spans { if span := OtelSpanConvertToSendSpan(ctx, spaceID, spans[i]); span != nil { result = append(result, span) @@ -273,19 +271,13 @@ func OtelSpansConvertToSendSpans(ctx context.Context, spaceID string, spans []*R return result } -func OtelSpanConvertToSendSpan(ctx context.Context, spaceID string, resourceScopeSpan *ResourceScopeSpan) *loop_span.Span { +func OtelSpanConvertToSendSpan(ctx context.Context, spaceID string, resourceScopeSpan *ResourceScopeSpan) *LoopSpan { if resourceScopeSpan == nil || resourceScopeSpan.Span == nil { return nil } span := resourceScopeSpan.Span - startTimeUnixNanoInt64, err := strconv.ParseInt(span.StartTimeUnixNano, 10, 64) - if err != nil { - logs.CtxError(ctx, "startTimeUnixNano convert to int64 failed err=%+v", err) - } - endTimeUnixNanoInt64, err := strconv.ParseInt(span.EndTimeUnixNano, 10, 64) - if err != nil { - logs.CtxError(ctx, "endTimeUnixNano convert to int64 failed err=%+v", err) - } + startTimeUnixNanoInt64, _ := strconv.ParseInt(span.StartTimeUnixNano, 10, 64) + endTimeUnixNanoInt64, _ := strconv.ParseInt(span.EndTimeUnixNano, 10, 64) attributeMap := make(map[string]*AnyValue) for _, spanAttribute := range span.Attributes { @@ -309,18 +301,18 @@ func OtelSpanConvertToSendSpan(ctx context.Context, spaceID string, resourceScop if srcValue == nil { continue } - conf, ok := fieldConfMap[fieldKey] + conf, ok := FieldConfMap[fieldKey] if !ok { continue } - switch conf.dataType { + switch conf.DataType { case dataTypeString, dataTypeDefault: value, ok := srcValue.(string) if !ok { continue } - if conf.isTag { + if conf.IsTag { tagsString[fieldKey] = value } else { switch fieldKey { @@ -338,7 +330,7 @@ func OtelSpanConvertToSendSpan(ctx context.Context, spaceID string, resourceScop if !ok { continue } - if conf.isTag { + if conf.IsTag { tagsLong[fieldKey] = value } case dataTypeBool: @@ -346,7 +338,7 @@ func OtelSpanConvertToSendSpan(ctx context.Context, spaceID string, resourceScop if !ok { continue } - if conf.isTag { + if conf.IsTag { tagsBool[fieldKey] = value } case dataTypeFloat64: @@ -354,7 +346,7 @@ func OtelSpanConvertToSendSpan(ctx context.Context, spaceID string, resourceScop if !ok { continue } - if conf.isTag { + if conf.IsTag { tagsDouble[fieldKey] = value } case dataTypeArrayString: @@ -362,7 +354,7 @@ func OtelSpanConvertToSendSpan(ctx context.Context, spaceID string, resourceScop if !ok { continue } - if conf.isTag { + if conf.IsTag { tagsString[fieldKey] = strings.Join(value, ",") } default: @@ -383,7 +375,7 @@ func OtelSpanConvertToSendSpan(ctx context.Context, spaceID string, resourceScop // set runtime calRuntime(systemTagsString, resourceScopeSpan) - result := &loop_span.Span{ + result := &LoopSpan{ StartTime: startTimeUnixNanoInt64 / 1000, SpanID: span.SpanId, ParentID: span.ParentSpanId, @@ -414,7 +406,7 @@ func OtelSpanConvertToSendSpan(ctx context.Context, spaceID string, resourceScop return result } -func setLogID(span *loop_span.Span) { +func setLogID(span *LoopSpan) { if span == nil || span.TagsString == nil { return } @@ -471,9 +463,7 @@ func calCallOptions(ctx context.Context, tagsDouble map[string]float64, tagsLong modelCallOption.Stop = strings.Split(stopSequences, ",") delete(tagsLong, "stop_sequences") bytes, err := sonic.Marshal(modelCallOption) - if err != nil { - logs.CtxError(ctx, "modelCallOption marshal failed err=%+v", err) - } else { + if err == nil { tagsString[tracespec.CallOptions] = string(bytes) } } @@ -573,7 +563,7 @@ func processAttributesAndEvents(ctx context.Context, attributeMap map[string]*An // for a certain field, process it gradually according to its value priority, // first processing the low priority ones, and then processing the high priority ones. - for fieldKey, conf := range fieldConfMap { + for fieldKey, conf := range FieldConfMap { var singleRes interface{} // attribute key attributeKeyRes := processAttributeKey(ctx, conf, attributeMap) @@ -610,11 +600,11 @@ func getSamePrefixAttributesMap(attributeMap map[string]*AnyValue, prefixKey str return samePrefixAttributesMap } -func processAttributeKey(ctx context.Context, conf fieldConf, attributeMap map[string]*AnyValue) interface{} { - if attributeKeys := conf.attributeKey; len(attributeKeys) > 0 { +func processAttributeKey(ctx context.Context, conf FieldConf, attributeMap map[string]*AnyValue) interface{} { + if attributeKeys := conf.AttributeKey; len(attributeKeys) > 0 { for _, key := range attributeKeys { if x, ok := attributeMap[key]; ok { - return getValueByDataType(x, conf.dataType) + return getValueByDataType(x, conf.DataType) } } } @@ -622,8 +612,8 @@ func processAttributeKey(ctx context.Context, conf fieldConf, attributeMap map[s return nil } -func processAttributePrefix(ctx context.Context, fieldKey string, conf fieldConf, attributeMap map[string]*AnyValue) string { - for _, attributePrefixKey := range conf.attributeKeyPrefix { +func processAttributePrefix(ctx context.Context, fieldKey string, conf FieldConf, attributeMap map[string]*AnyValue) string { + for _, attributePrefixKey := range conf.AttributeKeyPrefix { srcAttrAggrRes := aggregateAttributesByPrefix(attributeMap, attributePrefixKey) if srcAttrAggrRes == nil { continue @@ -659,33 +649,26 @@ func processAttributePrefix(ctx context.Context, fieldKey string, conf fieldConf case openInferenceAttributeModelInputMessages: // openInference input message srcInput, err := open_inference.ConvertToModelInput(srcAttrAggrRes) if err != nil { - logs.CtxWarn(ctx, "input ConvertToModelInput failed err=%+v", err) continue } // pack tools srcTools := aggregateAttributesByPrefix(attributeMap, openInferenceAttributeModelInputTools) toBeMarshalObject, err = open_inference.AddTools2ModelInput(srcInput, srcTools) if err != nil { - logs.CtxWarn(ctx, "openInference AddTools2ModelInput failed err=%+v", err) continue } case openInferenceAttributeModelOutputMessages: // openInference output message resObject, err := open_inference.ConvertToModelOutput(srcAttrAggrRes) - if err != nil { - logs.CtxWarn(ctx, "input ConvertToModelOutput failed err=%+v", err) - } else { + if err == nil { toBeMarshalObject = resObject } default: } tempBytes, err := sonic.Marshal(toBeMarshalObject) - if err != nil { - logs.CtxError(ctx, "input aggregateAttributes failed err=%+v", err) - } else { + if err == nil { return string(tempBytes) } - } return "" @@ -701,14 +684,14 @@ func aggregateAttributesByPrefix(attributeMap map[string]*AnyValue, attributePre return srcAttrAggrRes } -func processEvent(ctx context.Context, fieldKey string, conf fieldConf, events []*SpanEvent, attributeMap map[string]*AnyValue) string { - if len(events) == 0 || len(conf.eventName) == 0 { +func processEvent(ctx context.Context, fieldKey string, conf FieldConf, events []*SpanEvent, attributeMap map[string]*AnyValue) string { + if len(events) == 0 || len(conf.EventName) == 0 { return "" } eventSlice := make([]map[string]interface{}, 0) isAllOtelMessage := true // only otel standard message events require packaging on the outer layer, the rest are not included for _, event := range events { - if !slices.Contains(conf.eventName, event.Name) { + if !slices.Contains(conf.EventName, event.Name) { continue } if !slices.Contains(otelMessageEventNameMap, event.Name) { @@ -754,9 +737,7 @@ func processEvent(ctx context.Context, fieldKey string, conf fieldConf, events [ } } bytes, err := sonic.Marshal(toBeMarshalObject) - if err != nil { - logs.CtxError(ctx, "modelInputEventMessageSlice marshal failed err=%+v", err) - } else { + if err == nil { resBytes = bytes } @@ -775,9 +756,7 @@ func getModelTools(ctx context.Context, attributeMap map[string]*AnyValue) inter if fParam, ok := fMap["parameters"]; ok { if fParamStr, ok := fParam.(string); ok { tempParameter := make(map[string]interface{}, 0) - if err := sonic.UnmarshalString(fParamStr, &tempParameter); err != nil { - logs.CtxInfo(ctx, "getModelTools UnmarshalString failed err=%+v", err) - } else { + if err := sonic.UnmarshalString(fParamStr, &tempParameter); err == nil { fMap["parameters"] = tempParameter } } diff --git a/backend/modules/observability/domain/trace/entity/otel/otel_convert_test.go b/backend/modules/observability/lib/otel/otel_convert_test.go similarity index 96% rename from backend/modules/observability/domain/trace/entity/otel/otel_convert_test.go rename to backend/modules/observability/lib/otel/otel_convert_test.go index 0b967fd04..1ceafd7ea 100644 --- a/backend/modules/observability/domain/trace/entity/otel/otel_convert_test.go +++ b/backend/modules/observability/lib/otel/otel_convert_test.go @@ -8,7 +8,6 @@ import ( "encoding/json" "testing" - "github.com/coze-dev/coze-loop/backend/modules/observability/domain/trace/entity/loop_span" "github.com/coze-dev/cozeloop-go/spec/tracespec" "github.com/stretchr/testify/assert" semconv1_27_0 "go.opentelemetry.io/otel/semconv/v1.27.0" @@ -200,7 +199,7 @@ func TestSpanTypeMapping(t *testing.T) { func TestSetLogID(t *testing.T) { tests := []struct { name string - span *loop_span.Span + span *LoopSpan expectedLogID string shouldHaveLogID bool }{ @@ -212,7 +211,7 @@ func TestSetLogID(t *testing.T) { }, { name: "span with nil TagsString", - span: &loop_span.Span{ + span: &LoopSpan{ TagsString: nil, }, expectedLogID: "", @@ -220,7 +219,7 @@ func TestSetLogID(t *testing.T) { }, { name: "span with logid in TagsString", - span: &loop_span.Span{ + span: &LoopSpan{ TagsString: map[string]string{ "logid": "test-log-id", "other": "value", @@ -231,7 +230,7 @@ func TestSetLogID(t *testing.T) { }, { name: "span without logid in TagsString", - span: &loop_span.Span{ + span: &LoopSpan{ TagsString: map[string]string{ "other": "value", }, @@ -750,7 +749,7 @@ func TestProcessAttributesAndEvents(t *testing.T) { events: []*SpanEvent{}, validate: func(t *testing.T, result map[string]interface{}) { assert.NotNil(t, result) - // Should have entries for all fieldConfMap keys, but values might be nil + // Should have entries for all FieldConfMap keys, but values might be nil assert.Contains(t, result, "span_type") }, }, @@ -1206,23 +1205,23 @@ func TestProcessAttributeKey(t *testing.T) { tests := []struct { name string - conf fieldConf + conf FieldConf attributeMap map[string]*AnyValue expected interface{} }{ { name: "no attribute keys in config", - conf: fieldConf{ - attributeKey: []string{}, + conf: FieldConf{ + AttributeKey: []string{}, }, attributeMap: map[string]*AnyValue{}, expected: nil, }, { name: "attribute key found", - conf: fieldConf{ - attributeKey: []string{"test.key"}, - dataType: dataTypeString, + conf: FieldConf{ + AttributeKey: []string{"test.key"}, + DataType: dataTypeString, }, attributeMap: map[string]*AnyValue{ "test.key": {Value: &AnyValue_StringValue{StringValue: "test-value"}}, @@ -1231,9 +1230,9 @@ func TestProcessAttributeKey(t *testing.T) { }, { name: "attribute key not found", - conf: fieldConf{ - attributeKey: []string{"missing.key"}, - dataType: dataTypeString, + conf: FieldConf{ + AttributeKey: []string{"missing.key"}, + DataType: dataTypeString, }, attributeMap: map[string]*AnyValue{ "other.key": {Value: &AnyValue_StringValue{StringValue: "other-value"}}, @@ -1242,9 +1241,9 @@ func TestProcessAttributeKey(t *testing.T) { }, { name: "multiple attribute keys, first found", - conf: fieldConf{ - attributeKey: []string{"first.key", "second.key"}, - dataType: dataTypeString, + conf: FieldConf{ + AttributeKey: []string{"first.key", "second.key"}, + DataType: dataTypeString, }, attributeMap: map[string]*AnyValue{ "first.key": {Value: &AnyValue_StringValue{StringValue: "first-value"}}, @@ -1534,7 +1533,7 @@ func TestProcessAttributePrefix(t *testing.T) { tests := []struct { name string fieldKey string - conf fieldConf + conf FieldConf attributeMap map[string]*AnyValue expected string validate func(t *testing.T, result string) @@ -1542,8 +1541,8 @@ func TestProcessAttributePrefix(t *testing.T) { { name: "no attribute prefix keys", fieldKey: "test_field", - conf: fieldConf{ - attributeKeyPrefix: []string{}, + conf: FieldConf{ + AttributeKeyPrefix: []string{}, }, attributeMap: map[string]*AnyValue{}, expected: "", @@ -1551,8 +1550,8 @@ func TestProcessAttributePrefix(t *testing.T) { { name: "no matching attributes", fieldKey: "test_field", - conf: fieldConf{ - attributeKeyPrefix: []string{"non.existing.prefix"}, + conf: FieldConf{ + AttributeKeyPrefix: []string{"non.existing.prefix"}, }, attributeMap: map[string]*AnyValue{}, expected: "", @@ -1560,8 +1559,8 @@ func TestProcessAttributePrefix(t *testing.T) { { name: "basic attribute aggregation", fieldKey: "test_field", - conf: fieldConf{ - attributeKeyPrefix: []string{"test.prefix"}, + conf: FieldConf{ + AttributeKeyPrefix: []string{"test.prefix"}, }, attributeMap: map[string]*AnyValue{ "test.prefix.child1": {Value: &AnyValue_StringValue{StringValue: "value1"}}, @@ -1580,8 +1579,8 @@ func TestProcessAttributePrefix(t *testing.T) { { name: "GenAI completion output special process", fieldKey: tracespec.Output, - conf: fieldConf{ - attributeKeyPrefix: []string{string(semconv1_27_0.GenAICompletionKey)}, + conf: FieldConf{ + AttributeKeyPrefix: []string{string(semconv1_27_0.GenAICompletionKey)}, }, attributeMap: map[string]*AnyValue{ string(semconv1_27_0.GenAICompletionKey) + ".0.content": {Value: &AnyValue_StringValue{StringValue: "response1"}}, @@ -1611,8 +1610,8 @@ func TestProcessAttributePrefix(t *testing.T) { { name: "GenAI prompt input special process", fieldKey: tracespec.Input, - conf: fieldConf{ - attributeKeyPrefix: []string{string(semconv1_27_0.GenAIPromptKey)}, + conf: FieldConf{ + AttributeKeyPrefix: []string{string(semconv1_27_0.GenAIPromptKey)}, attributeHighLevelKey: []highLevelKeyRuleConf{{key: "messages", rule: highLevelKeyRuleMap}}, }, attributeMap: map[string]*AnyValue{ @@ -1634,8 +1633,8 @@ func TestProcessAttributePrefix(t *testing.T) { { name: "GenAI prompt input with tools special process", fieldKey: tracespec.Input, - conf: fieldConf{ - attributeKeyPrefix: []string{string(semconv1_27_0.GenAIPromptKey)}, + conf: FieldConf{ + AttributeKeyPrefix: []string{string(semconv1_27_0.GenAIPromptKey)}, attributeHighLevelKey: []highLevelKeyRuleConf{{key: "messages", rule: highLevelKeyRuleMap}}, }, attributeMap: map[string]*AnyValue{ @@ -1669,8 +1668,8 @@ func TestProcessAttributePrefix(t *testing.T) { { name: "OpenInference input messages special process", fieldKey: tracespec.Input, - conf: fieldConf{ - attributeKeyPrefix: []string{openInferenceAttributeModelInputMessages}, + conf: FieldConf{ + AttributeKeyPrefix: []string{openInferenceAttributeModelInputMessages}, }, attributeMap: map[string]*AnyValue{ openInferenceAttributeModelInputMessages + ".0.message.role": {Value: &AnyValue_StringValue{StringValue: "user"}}, @@ -1690,8 +1689,8 @@ func TestProcessAttributePrefix(t *testing.T) { { name: "OpenInference input messages with tools special process", fieldKey: tracespec.Input, - conf: fieldConf{ - attributeKeyPrefix: []string{openInferenceAttributeModelInputMessages}, + conf: FieldConf{ + AttributeKeyPrefix: []string{openInferenceAttributeModelInputMessages}, }, attributeMap: map[string]*AnyValue{ openInferenceAttributeModelInputMessages + ".0.message.role": {Value: &AnyValue_StringValue{StringValue: "user"}}, @@ -1713,8 +1712,8 @@ func TestProcessAttributePrefix(t *testing.T) { { name: "OpenInference output messages special process", fieldKey: tracespec.Output, - conf: fieldConf{ - attributeKeyPrefix: []string{openInferenceAttributeModelOutputMessages}, + conf: FieldConf{ + AttributeKeyPrefix: []string{openInferenceAttributeModelOutputMessages}, }, attributeMap: map[string]*AnyValue{ openInferenceAttributeModelOutputMessages + ".0.message.role": {Value: &AnyValue_StringValue{StringValue: "assistant"}}, @@ -1734,8 +1733,8 @@ func TestProcessAttributePrefix(t *testing.T) { { name: "default case no special processing", fieldKey: "custom_field", - conf: fieldConf{ - attributeKeyPrefix: []string{"custom.prefix"}, + conf: FieldConf{ + AttributeKeyPrefix: []string{"custom.prefix"}, }, attributeMap: map[string]*AnyValue{ "custom.prefix.child1": {Value: &AnyValue_StringValue{StringValue: "value1"}}, @@ -1754,8 +1753,8 @@ func TestProcessAttributePrefix(t *testing.T) { { name: "empty aggregation result", fieldKey: "test_field", - conf: fieldConf{ - attributeKeyPrefix: []string{"empty.prefix"}, + conf: FieldConf{ + AttributeKeyPrefix: []string{"empty.prefix"}, }, attributeMap: map[string]*AnyValue{ "empty.prefix": {Value: &AnyValue_StringValue{StringValue: ""}}, diff --git a/backend/modules/observability/domain/trace/entity/otel/otel_json_request.go b/backend/modules/observability/lib/otel/otel_json_request.go similarity index 98% rename from backend/modules/observability/domain/trace/entity/otel/otel_json_request.go rename to backend/modules/observability/lib/otel/otel_json_request.go index b5d8860b1..de59f43c7 100644 --- a/backend/modules/observability/domain/trace/entity/otel/otel_json_request.go +++ b/backend/modules/observability/lib/otel/otel_json_request.go @@ -10,8 +10,6 @@ import ( "github.com/bytedance/sonic" v1 "go.opentelemetry.io/proto/otlp/trace/v1" - - "github.com/coze-dev/coze-loop/backend/pkg/logs" ) // ExportTraceServiceRequest Internal struct, compared to PB struct: TraceID & SpanID & ParentSpanId is string, int64/uint64 -> string, can support otel json source data @@ -233,7 +231,6 @@ type KeyValueList struct { func (x *KeyValueList) String(ctx context.Context) string { marshalString, err := sonic.MarshalString(x) if err != nil { - logs.CtxError(ctx, "KeyValueList marshal failed err=%+v", err) return "" } return marshalString @@ -246,7 +243,6 @@ type ArrayValue struct { func (x *ArrayValue) String(ctx context.Context) string { marshalString, err := sonic.MarshalString(x) if err != nil { - logs.CtxError(ctx, "ArrayValue marshal failed err=%+v", err) return "" } return marshalString diff --git a/backend/modules/observability/domain/trace/entity/otel/otel_json_request_test.go b/backend/modules/observability/lib/otel/otel_json_request_test.go similarity index 100% rename from backend/modules/observability/domain/trace/entity/otel/otel_json_request_test.go rename to backend/modules/observability/lib/otel/otel_json_request_test.go diff --git a/backend/modules/observability/domain/trace/entity/otel/otel_pb2json.go b/backend/modules/observability/lib/otel/otel_pb2json.go similarity index 97% rename from backend/modules/observability/domain/trace/entity/otel/otel_pb2json.go rename to backend/modules/observability/lib/otel/otel_pb2json.go index 94827cc23..350a5fd3e 100644 --- a/backend/modules/observability/domain/trace/entity/otel/otel_pb2json.go +++ b/backend/modules/observability/lib/otel/otel_pb2json.go @@ -4,11 +4,9 @@ package otel import ( - "context" "encoding/hex" "strconv" - "github.com/coze-dev/coze-loop/backend/pkg/logs" v3 "go.opentelemetry.io/proto/otlp/collector/trace/v1" v2 "go.opentelemetry.io/proto/otlp/common/v1" v1 "go.opentelemetry.io/proto/otlp/trace/v1" @@ -35,7 +33,7 @@ func otelAnyValuePbToJson(src *v2.AnyValue) *AnyValue { case *v2.AnyValue_BytesValue: innerAnyValue.Value = &AnyValue_BytesValue{BytesValue: src.GetBytesValue()} default: - logs.CtxError(context.Background(), "OtelAnyValuePbToJson unknown type: %v", src.Value) + return innerAnyValue } return innerAnyValue } diff --git a/backend/modules/observability/domain/trace/entity/otel/otel_pb2json_test.go b/backend/modules/observability/lib/otel/otel_pb2json_test.go similarity index 100% rename from backend/modules/observability/domain/trace/entity/otel/otel_pb2json_test.go rename to backend/modules/observability/lib/otel/otel_pb2json_test.go diff --git a/backend/modules/observability/lib/otel/otel_span.go b/backend/modules/observability/lib/otel/otel_span.go new file mode 100644 index 000000000..625351910 --- /dev/null +++ b/backend/modules/observability/lib/otel/otel_span.go @@ -0,0 +1,40 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 + +package otel + +type ResourceScopeSpan struct { + Resource *Resource `json:"resource,omitempty"` + Scope *InstrumentationScope `json:"scope,omitempty"` + Span *Span `json:"span,omitempty"` +} + +type LoopSpan struct { + StartTime int64 `json:"start_time"` // us + SpanID string `json:"span_id"` + ParentID string `json:"parent_id"` + TraceID string `json:"trace_id"` + DurationMicros int64 `json:"duration_micros"` // us + CallType string `json:"call_type"` + PSM string `json:"psm"` + LogID string `json:"log_id"` + WorkspaceID string `json:"space_id"` + SpanName string `json:"span_name"` + SpanType string `json:"span_type"` + Method string `json:"method"` + StatusCode int32 `json:"status_code"` + Input string `json:"input"` + Output string `json:"output"` + ObjectStorage string `json:"object_storage"` + + SystemTagsString map[string]string `json:"system_tags_string"` + SystemTagsLong map[string]int64 `json:"system_tags_long"` + SystemTagsDouble map[string]float64 `json:"system_tags_double"` + + TagsString map[string]string `json:"tags_string"` + TagsLong map[string]int64 `json:"tags_long"` + TagsDouble map[string]float64 `json:"tags_double"` + + TagsBool map[string]bool `json:"tags_bool"` + TagsByte map[string]string `json:"tags_byte"` +} diff --git a/backend/modules/observability/domain/trace/entity/otel/otel_span_test.go b/backend/modules/observability/lib/otel/otel_span_test.go similarity index 100% rename from backend/modules/observability/domain/trace/entity/otel/otel_span_test.go rename to backend/modules/observability/lib/otel/otel_span_test.go diff --git a/idl/thrift/coze/loop/observability/domain/common.thrift b/idl/thrift/coze/loop/observability/domain/common.thrift index 13aacfa34..7df690132 100644 --- a/idl/thrift/coze/loop/observability/domain/common.thrift +++ b/idl/thrift/coze/loop/observability/domain/common.thrift @@ -10,6 +10,7 @@ const PlatformType PlatformType_Project = "coze_project" const PlatformType PlatformType_Workflow = "coze_workflow" const PlatformType PlatformType_Ark = "ark" const PlatformType PlatformType_VeADK = "veadk" +const PlatformType PlatformType_VeAgentkit = "ve_agentkit" const PlatformType PlatformType_LoopAll = "loop_all" const PlatformType PlatformType_InnerCozeloop = "inner_cozeloop" const PlatformType PlatformType_InnerDoubao = "inner_doubao"