From 20f863356438ccffcfafafec29f58e455daa9e3e Mon Sep 17 00:00:00 2001 From: swatimodi-scout Date: Mon, 4 Aug 2025 07:41:25 +0000 Subject: [PATCH 01/30] Added enpoint in configuration --- bindings/aws/kinesis/kinesis.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/bindings/aws/kinesis/kinesis.go b/bindings/aws/kinesis/kinesis.go index bf684f8bbb..0b84b44dcd 100644 --- a/bindings/aws/kinesis/kinesis.go +++ b/bindings/aws/kinesis/kinesis.go @@ -156,8 +156,15 @@ func (a *AWSKinesis) Read(ctx context.Context, handler bindings.Handler) (err er if a.closed.Load() { return errors.New("binding is closed") } + if a.metadata.KinesisConsumerMode == SharedThroughput { - a.worker = worker.NewWorker(a.recordProcessorFactory(ctx, handler), a.authProvider.Kinesis().WorkerCfg(ctx, a.streamName, a.consumerName, a.consumerMode)) + // Configure the KCL worker with custom endpoints for LocalStack + config := a.authProvider.Kinesis().WorkerCfg(ctx, a.streamName, a.consumerName, a.consumerMode) + if a.metadata.Endpoint != "" { + config.KinesisEndpoint = a.metadata.Endpoint + config.DynamoDBEndpoint = a.metadata.Endpoint + } + a.worker = worker.NewWorker(a.recordProcessorFactory(ctx, handler), config) err = a.worker.Start() if err != nil { return err From d891be89fee1a028b5f1f8946504b726a8128cf6 Mon Sep 17 00:00:00 2001 From: devendrapohekar-scout Date: Tue, 5 Aug 2025 13:34:02 +0530 Subject: [PATCH 02/30] fix: Handle 'extended' KinesisConsumerMode and prevent nil pointer errors - Invoke logic only when KinesisConsumerMode is set to 'extended' to avoid unnecessary calls - Skip next step if error is not nil to prevent nil pointer errors on StreamDescription.StreamARN. --- bindings/aws/kinesis/kinesis.go | 25 +++++++++++++++++-------- common/authentication/aws/client.go | 9 +++++++-- 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/bindings/aws/kinesis/kinesis.go b/bindings/aws/kinesis/kinesis.go index 0b84b44dcd..0b269e078d 100644 --- a/bindings/aws/kinesis/kinesis.go +++ b/bindings/aws/kinesis/kinesis.go @@ -156,8 +156,8 @@ func (a *AWSKinesis) Read(ctx context.Context, handler bindings.Handler) (err er if a.closed.Load() { return errors.New("binding is closed") } - - if a.metadata.KinesisConsumerMode == SharedThroughput { + switch a.metadata.KinesisConsumerMode { + case SharedThroughput: // Configure the KCL worker with custom endpoints for LocalStack config := a.authProvider.Kinesis().WorkerCfg(ctx, a.streamName, a.consumerName, a.consumerMode) if a.metadata.Endpoint != "" { @@ -169,7 +169,7 @@ func (a *AWSKinesis) Read(ctx context.Context, handler bindings.Handler) (err er if err != nil { return err } - } else if a.metadata.KinesisConsumerMode == ExtendedFanout { + case ExtendedFanout: var stream *kinesis.DescribeStreamOutput stream, err = a.authProvider.Kinesis().Kinesis.DescribeStream(&kinesis.DescribeStreamInput{StreamName: &a.metadata.StreamName}) if err != nil { @@ -181,10 +181,18 @@ func (a *AWSKinesis) Read(ctx context.Context, handler bindings.Handler) (err er } } - stream, err := a.authProvider.Kinesis().Stream(ctx, a.streamName) - if err != nil { - return fmt.Errorf("failed to get kinesis stream arn: %v", err) + var stream *string + /** + * Invoke this only when KinesisConsumerMode is set to 'extended' to avoid unnecessary calls. + */ + if a.metadata.KinesisConsumerMode == ExtendedFanout { + streamARN, err := a.authProvider.Kinesis().Stream(ctx, a.streamName) + if err != nil { + return fmt.Errorf("failed to get kinesis stream arn: %v", err) + } + stream = streamARN } + // Wait for context cancelation then stop a.wg.Add(1) go func() { @@ -193,9 +201,10 @@ func (a *AWSKinesis) Read(ctx context.Context, handler bindings.Handler) (err er case <-ctx.Done(): case <-a.closeCh: } - if a.metadata.KinesisConsumerMode == SharedThroughput { + switch a.metadata.KinesisConsumerMode { + case SharedThroughput: a.worker.Shutdown() - } else if a.metadata.KinesisConsumerMode == ExtendedFanout { + case ExtendedFanout: a.deregisterConsumer(ctx, stream, a.consumerARN) } }() diff --git a/common/authentication/aws/client.go b/common/authentication/aws/client.go index 11b26e4988..08a22991c5 100644 --- a/common/authentication/aws/client.go +++ b/common/authentication/aws/client.go @@ -205,9 +205,14 @@ func (c *KinesisClients) Stream(ctx context.Context, streamName string) (*string stream, err := c.Kinesis.DescribeStreamWithContext(ctx, &kinesis.DescribeStreamInput{ StreamName: aws.String(streamName), }) - if stream != nil { - return stream.StreamDescription.StreamARN, err + /** + * If the error is not nil, do not proceed to the next step + * as it may cause a nil pointer error on stream.StreamDescription.StreamARN. + */ + if err != nil { + return nil, err } + return stream.StreamDescription.StreamARN, err } return nil, errors.New("unable to get stream arn due to empty client") From 281c7bd3f3aba0c2094456146171bb9d75b637ae Mon Sep 17 00:00:00 2001 From: devendrapohekar-scout Date: Tue, 5 Aug 2025 14:47:39 +0530 Subject: [PATCH 03/30] fix: update comments. --- bindings/aws/kinesis/kinesis.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/bindings/aws/kinesis/kinesis.go b/bindings/aws/kinesis/kinesis.go index 0b269e078d..32c7b01829 100644 --- a/bindings/aws/kinesis/kinesis.go +++ b/bindings/aws/kinesis/kinesis.go @@ -156,8 +156,7 @@ func (a *AWSKinesis) Read(ctx context.Context, handler bindings.Handler) (err er if a.closed.Load() { return errors.New("binding is closed") } - switch a.metadata.KinesisConsumerMode { - case SharedThroughput: + if a.metadata.KinesisConsumerMode == SharedThroughput { // Configure the KCL worker with custom endpoints for LocalStack config := a.authProvider.Kinesis().WorkerCfg(ctx, a.streamName, a.consumerName, a.consumerMode) if a.metadata.Endpoint != "" { @@ -169,7 +168,7 @@ func (a *AWSKinesis) Read(ctx context.Context, handler bindings.Handler) (err er if err != nil { return err } - case ExtendedFanout: + } else if a.metadata.KinesisConsumerMode == ExtendedFanout { var stream *kinesis.DescribeStreamOutput stream, err = a.authProvider.Kinesis().Kinesis.DescribeStream(&kinesis.DescribeStreamInput{StreamName: &a.metadata.StreamName}) if err != nil { @@ -201,10 +200,9 @@ func (a *AWSKinesis) Read(ctx context.Context, handler bindings.Handler) (err er case <-ctx.Done(): case <-a.closeCh: } - switch a.metadata.KinesisConsumerMode { - case SharedThroughput: + if a.metadata.KinesisConsumerMode == SharedThroughput { a.worker.Shutdown() - case ExtendedFanout: + } else if a.metadata.KinesisConsumerMode == ExtendedFanout { a.deregisterConsumer(ctx, stream, a.consumerARN) } }() From e25da2872cde64eeb3b13523a21a8f04bad3cdca Mon Sep 17 00:00:00 2001 From: devendrapohekar-scout Date: Wed, 6 Aug 2025 18:24:52 +0530 Subject: [PATCH 04/30] feat: upgradevmware-go-kcl to vmware-go-kcl-v2. --- bindings/aws/kinesis/kinesis.go | 59 ++++++++++-------------- bindings/aws/kinesis/kinesis_test.go | 18 ++++---- common/authentication/aws/client.go | 18 +++----- common/authentication/aws/client_test.go | 43 +++++++++-------- go.mod | 9 ++-- go.sum | 21 +++++---- 6 files changed, 81 insertions(+), 87 deletions(-) diff --git a/bindings/aws/kinesis/kinesis.go b/bindings/aws/kinesis/kinesis.go index 32c7b01829..8c686f5c15 100644 --- a/bindings/aws/kinesis/kinesis.go +++ b/bindings/aws/kinesis/kinesis.go @@ -23,12 +23,13 @@ import ( "time" "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/request" "github.com/aws/aws-sdk-go/service/kinesis" "github.com/cenkalti/backoff/v4" "github.com/google/uuid" - "github.com/vmware/vmware-go-kcl/clientlibrary/interfaces" - "github.com/vmware/vmware-go-kcl/clientlibrary/worker" + "github.com/vmware/vmware-go-kcl-v2/clientlibrary/interfaces" + "github.com/vmware/vmware-go-kcl-v2/clientlibrary/worker" "github.com/dapr/components-contrib/bindings" awsAuth "github.com/dapr/components-contrib/common/authentication/aws" @@ -44,11 +45,12 @@ type AWSKinesis struct { worker *worker.Worker - streamName string - consumerName string - consumerARN *string - logger logger.Logger - consumerMode string + streamName string + consumerName string + consumerARN *string + logger logger.Logger + consumerMode string + applicationName string closed atomic.Bool closeCh chan struct{} @@ -64,6 +66,7 @@ type kinesisMetadata struct { SecretKey string `json:"secretKey" mapstructure:"secretKey"` SessionToken string `json:"sessionToken" mapstructure:"sessionToken"` KinesisConsumerMode string `json:"mode" mapstructure:"mode"` + ApplicationName string `json:"applicationName" mapstructure:"applicationName"` } const ( @@ -116,6 +119,7 @@ func (a *AWSKinesis) Init(ctx context.Context, metadata bindings.Metadata) error a.streamName = m.StreamName a.consumerName = m.ConsumerName a.metadata = m + a.applicationName = m.ApplicationName opts := awsAuth.Options{ Logger: a.logger, @@ -156,42 +160,29 @@ func (a *AWSKinesis) Read(ctx context.Context, handler bindings.Handler) (err er if a.closed.Load() { return errors.New("binding is closed") } - if a.metadata.KinesisConsumerMode == SharedThroughput { + // initalize worker configuration + config := a.authProvider.Kinesis().WorkerCfg(ctx, a.streamName, a.metadata.Region, a.consumerMode, a.applicationName) + config.WithKinesisEndpoint(a.metadata.Endpoint) + config.WithDynamoDBEndpoint(a.metadata.Endpoint) + + switch a.metadata.KinesisConsumerMode { + case SharedThroughput: // Configure the KCL worker with custom endpoints for LocalStack - config := a.authProvider.Kinesis().WorkerCfg(ctx, a.streamName, a.consumerName, a.consumerMode) - if a.metadata.Endpoint != "" { - config.KinesisEndpoint = a.metadata.Endpoint - config.DynamoDBEndpoint = a.metadata.Endpoint - } a.worker = worker.NewWorker(a.recordProcessorFactory(ctx, handler), config) err = a.worker.Start() if err != nil { return err } - } else if a.metadata.KinesisConsumerMode == ExtendedFanout { - var stream *kinesis.DescribeStreamOutput - stream, err = a.authProvider.Kinesis().Kinesis.DescribeStream(&kinesis.DescribeStreamInput{StreamName: &a.metadata.StreamName}) - if err != nil { - return err - } - err = a.Subscribe(ctx, *stream.StreamDescription, handler) + case ExtendedFanout: + config.WithEnhancedFanOutConsumer(true) + config.WithEnhancedFanOutConsumerName(a.consumerName) + a.worker = worker.NewWorker(a.recordProcessorFactory(ctx, handler), config) + err := a.worker.Start() if err != nil { return err } } - var stream *string - /** - * Invoke this only when KinesisConsumerMode is set to 'extended' to avoid unnecessary calls. - */ - if a.metadata.KinesisConsumerMode == ExtendedFanout { - streamARN, err := a.authProvider.Kinesis().Stream(ctx, a.streamName) - if err != nil { - return fmt.Errorf("failed to get kinesis stream arn: %v", err) - } - stream = streamARN - } - // Wait for context cancelation then stop a.wg.Add(1) go func() { @@ -200,10 +191,8 @@ func (a *AWSKinesis) Read(ctx context.Context, handler bindings.Handler) (err er case <-ctx.Done(): case <-a.closeCh: } - if a.metadata.KinesisConsumerMode == SharedThroughput { + if a.worker != nil { a.worker.Shutdown() - } else if a.metadata.KinesisConsumerMode == ExtendedFanout { - a.deregisterConsumer(ctx, stream, a.consumerARN) } }() diff --git a/bindings/aws/kinesis/kinesis_test.go b/bindings/aws/kinesis/kinesis_test.go index aaca0c3c0a..154e9baa57 100644 --- a/bindings/aws/kinesis/kinesis_test.go +++ b/bindings/aws/kinesis/kinesis_test.go @@ -25,14 +25,15 @@ import ( func TestParseMetadata(t *testing.T) { m := bindings.Metadata{} m.Properties = map[string]string{ - "accessKey": "key", - "region": "region", - "secretKey": "secret", - "consumerName": "test", - "streamName": "stream", - "mode": "extended", - "endpoint": "endpoint", - "sessionToken": "token", + "accessKey": "key", + "region": "region", + "secretKey": "secret", + "consumerName": "test", + "streamName": "stream", + "mode": "extended", + "endpoint": "endpoint", + "sessionToken": "token", + "applicationName": "applicationName", } kinesis := AWSKinesis{} meta, err := kinesis.parseMetadata(m) @@ -45,4 +46,5 @@ func TestParseMetadata(t *testing.T) { assert.Equal(t, "endpoint", meta.Endpoint) assert.Equal(t, "token", meta.SessionToken) assert.Equal(t, "extended", meta.KinesisConsumerMode) + assert.Equal(t, "applicationName", meta.ApplicationName) } diff --git a/common/authentication/aws/client.go b/common/authentication/aws/client.go index 08a22991c5..4d865c6115 100644 --- a/common/authentication/aws/client.go +++ b/common/authentication/aws/client.go @@ -24,7 +24,6 @@ import ( "github.com/aws/aws-msk-iam-sasl-signer-go/signer" aws2 "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/dynamodb" "github.com/aws/aws-sdk-go/service/dynamodb/dynamodbiface" @@ -41,7 +40,7 @@ import ( "github.com/aws/aws-sdk-go/service/ssm" "github.com/aws/aws-sdk-go/service/ssm/ssmiface" "github.com/aws/aws-sdk-go/service/sts" - "github.com/vmware/vmware-go-kcl/clientlibrary/config" + "github.com/vmware/vmware-go-kcl-v2/clientlibrary/config" ) type Clients struct { @@ -132,7 +131,7 @@ type ParameterStoreClients struct { type KinesisClients struct { Kinesis kinesisiface.KinesisAPI Region string - Credentials *credentials.Credentials + Credentials aws2.CredentialsProvider } type SesClients struct { @@ -197,7 +196,6 @@ func (c *ParameterStoreClients) New(session *session.Session) { func (c *KinesisClients) New(session *session.Session) { c.Kinesis = kinesis.New(session, session.Config) c.Region = *session.Config.Region - c.Credentials = session.Config.Credentials } func (c *KinesisClients) Stream(ctx context.Context, streamName string) (*string, error) { @@ -218,19 +216,15 @@ func (c *KinesisClients) Stream(ctx context.Context, streamName string) (*string return nil, errors.New("unable to get stream arn due to empty client") } -func (c *KinesisClients) WorkerCfg(ctx context.Context, stream, consumer, mode string) *config.KinesisClientLibConfiguration { +func (c *KinesisClients) WorkerCfg(ctx context.Context, stream, region, mode, applicationName string) *config.KinesisClientLibConfiguration { const sharedMode = "shared" if c.Kinesis != nil { if mode == sharedMode { - if c.Credentials != nil { - kclConfig := config.NewKinesisClientLibConfigWithCredential(consumer, - stream, c.Region, consumer, - c.Credentials) - return kclConfig - } + kclConfig := config.NewKinesisClientLibConfigWithCredential(applicationName, stream, region, "", nil) + return kclConfig + } } - return nil } diff --git a/common/authentication/aws/client_test.go b/common/authentication/aws/client_test.go index 85e0392aae..628fbb320a 100644 --- a/common/authentication/aws/client_test.go +++ b/common/authentication/aws/client_test.go @@ -19,6 +19,7 @@ import ( "testing" "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/request" "github.com/aws/aws-sdk-go/aws/session" @@ -28,7 +29,7 @@ import ( "github.com/aws/aws-sdk-go/service/sqs/sqsiface" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/vmware/vmware-go-kcl/clientlibrary/config" + "github.com/vmware/vmware-go-kcl-v2/clientlibrary/config" ) type mockedSQS struct { @@ -186,12 +187,13 @@ func TestKinesisClients_Stream(t *testing.T) { func TestKinesisClients_WorkerCfg(t *testing.T) { testCreds := credentials.NewStaticCredentials("accessKey", "secretKey", "") tests := []struct { - name string - kinesisClient *KinesisClients - streamName string - consumer string - mode string - expectedConfig *config.KinesisClientLibConfiguration + name string + kinesisClient *KinesisClients + streamName string + consumer string + applicationName string + mode string + expectedConfig *config.KinesisClientLibConfiguration }{ { name: "successfully creates shared mode worker config", @@ -208,9 +210,10 @@ func TestKinesisClients_WorkerCfg(t *testing.T) { Region: "us-west-1", Credentials: testCreds, }, - streamName: "existing-stream", - consumer: "consumer1", - mode: "shared", + streamName: "existing-stream", + applicationName: "test-application", + consumer: "consumer1", + mode: "shared", expectedConfig: config.NewKinesisClientLibConfigWithCredential( "consumer1", "existing-stream", "us-west-1", "consumer1", testCreds, ), @@ -230,10 +233,11 @@ func TestKinesisClients_WorkerCfg(t *testing.T) { Region: "us-west-1", Credentials: testCreds, }, - streamName: "existing-stream", - consumer: "consumer1", - mode: "exclusive", - expectedConfig: nil, + streamName: "existing-stream", + applicationName: "test-application", + consumer: "consumer1", + mode: "exclusive", + expectedConfig: nil, }, { name: "returns nil when client is nil", @@ -242,16 +246,17 @@ func TestKinesisClients_WorkerCfg(t *testing.T) { Region: "us-west-1", Credentials: credentials.NewStaticCredentials("accessKey", "secretKey", ""), }, - streamName: "existing-stream", - consumer: "consumer1", - mode: "shared", - expectedConfig: nil, + streamName: "existing-stream", + applicationName: "test-application", + consumer: "consumer1", + mode: "shared", + expectedConfig: nil, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - cfg := tt.kinesisClient.WorkerCfg(t.Context(), tt.streamName, tt.consumer, tt.mode) + cfg := tt.kinesisClient.WorkerCfg(t.Context(), tt.streamName, tt.mode, tt.applicationName) if tt.expectedConfig == nil { assert.Equal(t, tt.expectedConfig, cfg) return diff --git a/go.mod b/go.mod index 7ca8ed109b..c8a5756446 100644 --- a/go.mod +++ b/go.mod @@ -41,7 +41,7 @@ require ( github.com/apache/thrift v0.13.0 github.com/aws/aws-msk-iam-sasl-signer-go v1.0.1-0.20241125194140-078c08b8574a github.com/aws/aws-sdk-go v1.55.6 - github.com/aws/aws-sdk-go-v2 v1.36.5 + github.com/aws/aws-sdk-go-v2 v1.37.2 github.com/aws/aws-sdk-go-v2/config v1.29.17 github.com/aws/aws-sdk-go-v2/credentials v1.17.70 github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.19.3 @@ -125,7 +125,7 @@ require ( github.com/tetratelabs/wazero v1.7.0 github.com/tmc/langchaingo v0.1.13 github.com/valyala/fasthttp v1.53.0 - github.com/vmware/vmware-go-kcl v1.5.1 + github.com/vmware/vmware-go-kcl-v2 v1.0.0 github.com/xdg-go/scram v1.1.2 go.etcd.io/etcd/client/v3 v3.5.21 go.mongodb.org/mongo-driver v1.14.0 @@ -208,10 +208,11 @@ require ( github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.10.17 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.17 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.17 // indirect + github.com/aws/aws-sdk-go-v2/service/kinesis v1.27.1 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.25.5 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.3 // indirect - github.com/aws/smithy-go v1.22.4 // indirect - github.com/awslabs/kinesis-aggregation/go v0.0.0-20210630091500-54e17340d32f // indirect + github.com/aws/smithy-go v1.22.5 // indirect + github.com/awslabs/kinesis-aggregation/go/v2 v2.0.0-20211222152315-953b66f67407 // indirect github.com/benbjohnson/clock v1.3.5 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bits-and-blooms/bitset v1.4.0 // indirect diff --git a/go.sum b/go.sum index 8fae8aa1c8..f49be2ff3e 100644 --- a/go.sum +++ b/go.sum @@ -274,15 +274,15 @@ github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:W github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= github.com/aws/aws-msk-iam-sasl-signer-go v1.0.1-0.20241125194140-078c08b8574a h1:QFemvMGPnajaeRBkFc1HoEA7qzVjUv+rkYb1/ps1/UE= github.com/aws/aws-msk-iam-sasl-signer-go v1.0.1-0.20241125194140-078c08b8574a/go.mod h1:MVYeeOhILFFemC/XlYTClvBjYZrg/EPd3ts885KrNTI= -github.com/aws/aws-sdk-go v1.19.48/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.32.6/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/aws/aws-sdk-go v1.55.6 h1:cSg4pvZ3m8dgYcgqB97MrcdjUmZ1BeMYKUxMMB89IPk= github.com/aws/aws-sdk-go v1.55.6/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= +github.com/aws/aws-sdk-go-v2 v1.9.0/go.mod h1:cK/D0BBs0b/oWPIcX/Z/obahJK1TT7IPVjy53i/mX/4= github.com/aws/aws-sdk-go-v2 v1.9.2/go.mod h1:cK/D0BBs0b/oWPIcX/Z/obahJK1TT7IPVjy53i/mX/4= -github.com/aws/aws-sdk-go-v2 v1.36.5 h1:0OF9RiEMEdDdZEMqF9MRjevyxAQcf6gY+E7vwBILFj0= -github.com/aws/aws-sdk-go-v2 v1.36.5/go.mod h1:EYrzvCCN9CMUTa5+6lf6MM4tq3Zjp8UhSGR/cBsjai0= +github.com/aws/aws-sdk-go-v2 v1.37.2 h1:xkW1iMYawzcmYFYEV0UCMxc8gSsjCGEhBXQkdQywVbo= +github.com/aws/aws-sdk-go-v2 v1.37.2/go.mod h1:9Q0OoGQoboYIAJyslFyF1f5K1Ryddop8gqMhWx/n4Wg= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.11 h1:12SpdwU8Djs+YGklkinSSlcrPyj3H4VifVsKf78KbwA= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.11/go.mod h1:dd+Lkp6YmMryke+qxW/VnKyhMBDTYP41Q2Bb+6gNZgY= github.com/aws/aws-sdk-go-v2/config v1.8.3/go.mod h1:4AEiLtAb8kLs7vgw2ZV3p2VZ1+hBavOc84hqxVNpCyw= @@ -327,6 +327,9 @@ github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.17 h1:t0E6FzRE github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.17/go.mod h1:ygpklyoaypuyDvOM5ujWGrYWpAK3h7ugnmKCU/76Ys4= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.17 h1:qcLWgdhq45sDM9na4cvXax9dyLitn8EYBRl8Ak4XtG4= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.17/go.mod h1:M+jkjBFZ2J6DJrjMv2+vkBbuht6kxJYtJiwoVgX4p4U= +github.com/aws/aws-sdk-go-v2/service/kinesis v1.6.0/go.mod h1:9O7UG2pELnP0hq35+Gd7XDjOLBkg7tmgRQ0y14ZjoJI= +github.com/aws/aws-sdk-go-v2/service/kinesis v1.27.1 h1:p8dOJ/UKXOwttc1Cxw1Ek52klVmMuiaCUkhsUGxce1I= +github.com/aws/aws-sdk-go-v2/service/kinesis v1.27.1/go.mod h1:VpH1IBG1YYZHPu5qShNt7EGaqUQbHAJZrbDtEpqDvvY= github.com/aws/aws-sdk-go-v2/service/s3 v1.83.0 h1:5Y75q0RPQoAbieyOuGLhjV9P3txvYgXv2lg0UwJOfmE= github.com/aws/aws-sdk-go-v2/service/s3 v1.83.0/go.mod h1:kUklwasNoCn5YpyAqC/97r6dzTA1SRKJfKq16SXeoDU= github.com/aws/aws-sdk-go-v2/service/sns v1.34.7 h1:OBuZE9Wt8h2imuRktu+WfjiTGrnYdCIJg8IX92aalHE= @@ -344,10 +347,10 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.34.0/go.mod h1:7ph2tGpfQvwzgistp2+zg github.com/aws/rolesanywhere-credential-helper v1.0.4 h1:kHIVVdyQQiFZoKBP+zywBdFilGCS8It+UvW5LolKbW8= github.com/aws/rolesanywhere-credential-helper v1.0.4/go.mod h1:QVGNxlDlYhjR0/ZUee7uGl0hNChWidNpe2+GD87Buqk= github.com/aws/smithy-go v1.8.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= -github.com/aws/smithy-go v1.22.4 h1:uqXzVZNuNexwc/xrh6Tb56u89WDlJY6HS+KC0S4QSjw= -github.com/aws/smithy-go v1.22.4/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI= -github.com/awslabs/kinesis-aggregation/go v0.0.0-20210630091500-54e17340d32f h1:Pf0BjJDga7C98f0vhw+Ip5EaiE07S3lTKpIYPNS0nMo= -github.com/awslabs/kinesis-aggregation/go v0.0.0-20210630091500-54e17340d32f/go.mod h1:SghidfnxvX7ribW6nHI7T+IBbc9puZ9kk5Tx/88h8P4= +github.com/aws/smithy-go v1.22.5 h1:P9ATCXPMb2mPjYBgueqJNCA5S9UfktsW0tTxi+a7eqw= +github.com/aws/smithy-go v1.22.5/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI= +github.com/awslabs/kinesis-aggregation/go/v2 v2.0.0-20211222152315-953b66f67407 h1:p8Ubi4GEgfRc1xFn/WtGNkVG8RXxGHOsKiwGptufIo8= +github.com/awslabs/kinesis-aggregation/go/v2 v2.0.0-20211222152315-953b66f67407/go.mod h1:0Qr1uMHFmHsIYMcG4T7BJ9yrJtWadhOmpABCX69dwuc= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= @@ -1728,8 +1731,8 @@ github.com/valyala/fasthttp v1.53.0 h1:lW/+SUkOxCx2vlIu0iaImv4JLrVRnbbkpCoaawvA4 github.com/valyala/fasthttp v1.53.0/go.mod h1:6dt4/8olwq9QARP/TDuPmWyWcl4byhpvTJ4AAtcz+QM= github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= -github.com/vmware/vmware-go-kcl v1.5.1 h1:1rJLfAX4sDnCyatNoD/WJzVafkwST6u/cgY/Uf2VgHk= -github.com/vmware/vmware-go-kcl v1.5.1/go.mod h1:kXJmQ6h0dRMRrp1uWU9XbIXvwelDpTxSPquvQUBdpbo= +github.com/vmware/vmware-go-kcl-v2 v1.0.0 h1:HPT5vu+khRmGspBSc/+AilEWbRGoTZhjlYqdrBbRMZs= +github.com/vmware/vmware-go-kcl-v2 v1.0.0/go.mod h1:GBDu+P4Neo0vwZAk0ZUCEC8GYsUOWvi3XhFwAZR3SjA= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= From 792c790a1dd04de973a4438177559f534e180d19 Mon Sep 17 00:00:00 2001 From: devendrapohekar-scout Date: Wed, 6 Aug 2025 18:50:09 +0530 Subject: [PATCH 05/30] feat: update endpoint and revert back some changes. --- bindings/aws/kinesis/kinesis.go | 36 ++++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/bindings/aws/kinesis/kinesis.go b/bindings/aws/kinesis/kinesis.go index 8c686f5c15..df01e7663d 100644 --- a/bindings/aws/kinesis/kinesis.go +++ b/bindings/aws/kinesis/kinesis.go @@ -160,29 +160,45 @@ func (a *AWSKinesis) Read(ctx context.Context, handler bindings.Handler) (err er if a.closed.Load() { return errors.New("binding is closed") } - // initalize worker configuration - config := a.authProvider.Kinesis().WorkerCfg(ctx, a.streamName, a.metadata.Region, a.consumerMode, a.applicationName) - config.WithKinesisEndpoint(a.metadata.Endpoint) - config.WithDynamoDBEndpoint(a.metadata.Endpoint) switch a.metadata.KinesisConsumerMode { case SharedThroughput: + // initalize worker configuration + config := a.authProvider.Kinesis().WorkerCfg(ctx, a.streamName, a.metadata.Region, a.consumerMode, a.applicationName) // Configure the KCL worker with custom endpoints for LocalStack + if a.metadata.Endpoint != "" { + config.WithKinesisEndpoint(a.metadata.Endpoint) + config.WithDynamoDBEndpoint(a.metadata.Endpoint) + } a.worker = worker.NewWorker(a.recordProcessorFactory(ctx, handler), config) err = a.worker.Start() if err != nil { return err } case ExtendedFanout: - config.WithEnhancedFanOutConsumer(true) - config.WithEnhancedFanOutConsumerName(a.consumerName) - a.worker = worker.NewWorker(a.recordProcessorFactory(ctx, handler), config) - err := a.worker.Start() + var stream *kinesis.DescribeStreamOutput + stream, err = a.authProvider.Kinesis().Kinesis.DescribeStream(&kinesis.DescribeStreamInput{StreamName: &a.metadata.StreamName}) + if err != nil { + return err + } + err = a.Subscribe(ctx, *stream.StreamDescription, handler) if err != nil { return err } } + var stream *string + /** + * Invoke this only when KinesisConsumerMode is set to 'extended' to avoid unnecessary calls. + */ + if a.metadata.KinesisConsumerMode == ExtendedFanout { + streamARN, err := a.authProvider.Kinesis().Stream(ctx, a.streamName) + if err != nil { + return fmt.Errorf("failed to get kinesis stream arn: %v", err) + } + stream = streamARN + } + // Wait for context cancelation then stop a.wg.Add(1) go func() { @@ -191,8 +207,10 @@ func (a *AWSKinesis) Read(ctx context.Context, handler bindings.Handler) (err er case <-ctx.Done(): case <-a.closeCh: } - if a.worker != nil { + if a.metadata.KinesisConsumerMode == SharedThroughput { a.worker.Shutdown() + } else if a.metadata.KinesisConsumerMode == ExtendedFanout { + a.deregisterConsumer(ctx, stream, a.consumerARN) } }() From df6c7f1b9df35b087cab1f3645e6ee073a207bfb Mon Sep 17 00:00:00 2001 From: Javier Aliaga Date: Fri, 1 Aug 2025 18:14:50 +0200 Subject: [PATCH 06/30] chore: Use base64 to store sqlserver state data (#3919) Signed-off-by: Javier Aliaga Co-authored-by: Yaron Schneider Co-authored-by: Cassie Coyle Signed-off-by: swatimodi-scout --- common/proto/state/sqlserver/test.pb.go | 162 ++++++++++++++++++ common/proto/state/sqlserver/test.proto | 10 ++ state/sqlserver/sqlserver.go | 34 +++- state/sqlserver/sqlserver_integration_test.go | 30 +++- 4 files changed, 225 insertions(+), 11 deletions(-) create mode 100644 common/proto/state/sqlserver/test.pb.go create mode 100644 common/proto/state/sqlserver/test.proto diff --git a/common/proto/state/sqlserver/test.pb.go b/common/proto/state/sqlserver/test.pb.go new file mode 100644 index 0000000000..1c936adb57 --- /dev/null +++ b/common/proto/state/sqlserver/test.pb.go @@ -0,0 +1,162 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.32.0 +// protoc v4.25.4 +// source: test.proto + +package sqlserver + +import ( + "reflect" + "sync" + + "google.golang.org/protobuf/reflect/protoreflect" + "google.golang.org/protobuf/runtime/protoimpl" + "google.golang.org/protobuf/types/known/timestamppb" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type TestEvent struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + EventId int32 `protobuf:"varint,1,opt,name=eventId,proto3" json:"eventId,omitempty"` + Timestamp *timestamppb.Timestamp `protobuf:"bytes,2,opt,name=timestamp,proto3" json:"timestamp,omitempty"` +} + +func (x *TestEvent) Reset() { + *x = TestEvent{} + if protoimpl.UnsafeEnabled { + mi := &file_test_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *TestEvent) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TestEvent) ProtoMessage() {} + +func (x *TestEvent) ProtoReflect() protoreflect.Message { + mi := &file_test_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use TestEvent.ProtoReflect.Descriptor instead. +func (*TestEvent) Descriptor() ([]byte, []int) { + return file_test_proto_rawDescGZIP(), []int{0} +} + +func (x *TestEvent) GetEventId() int32 { + if x != nil { + return x.EventId + } + return 0 +} + +func (x *TestEvent) GetTimestamp() *timestamppb.Timestamp { + if x != nil { + return x.Timestamp + } + return nil +} + +var File_test_proto protoreflect.FileDescriptor + +var file_test_proto_rawDesc = []byte{ + 0x0a, 0x0a, 0x74, 0x65, 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1f, 0x67, 0x6f, + 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x74, 0x69, + 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x5f, 0x0a, + 0x09, 0x54, 0x65, 0x73, 0x74, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x65, 0x76, + 0x65, 0x6e, 0x74, 0x49, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x07, 0x65, 0x76, 0x65, + 0x6e, 0x74, 0x49, 0x64, 0x12, 0x38, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, + 0x70, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, + 0x61, 0x6d, 0x70, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x42, 0x41, + 0x5a, 0x3f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x64, 0x61, 0x70, + 0x72, 0x2f, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x73, 0x2d, 0x63, 0x6f, 0x6e, + 0x74, 0x72, 0x69, 0x62, 0x2f, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2f, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x2f, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2f, 0x73, 0x71, 0x6c, 0x73, 0x65, 0x72, 0x76, 0x65, + 0x72, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_test_proto_rawDescOnce sync.Once + file_test_proto_rawDescData = file_test_proto_rawDesc +) + +func file_test_proto_rawDescGZIP() []byte { + file_test_proto_rawDescOnce.Do(func() { + file_test_proto_rawDescData = protoimpl.X.CompressGZIP(file_test_proto_rawDescData) + }) + return file_test_proto_rawDescData +} + +var file_test_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_test_proto_goTypes = []interface{}{ + (*TestEvent)(nil), // 0: TestEvent + (*timestamppb.Timestamp)(nil), // 1: google.protobuf.Timestamp +} +var file_test_proto_depIdxs = []int32{ + 1, // 0: TestEvent.timestamp:type_name -> google.protobuf.Timestamp + 1, // [1:1] is the sub-list for method output_type + 1, // [1:1] is the sub-list for method input_type + 1, // [1:1] is the sub-list for extension type_name + 1, // [1:1] is the sub-list for extension extendee + 0, // [0:1] is the sub-list for field type_name +} + +func init() { file_test_proto_init() } +func file_test_proto_init() { + if File_test_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_test_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*TestEvent); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_test_proto_rawDesc, + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_test_proto_goTypes, + DependencyIndexes: file_test_proto_depIdxs, + MessageInfos: file_test_proto_msgTypes, + }.Build() + File_test_proto = out.File + file_test_proto_rawDesc = nil + file_test_proto_goTypes = nil + file_test_proto_depIdxs = nil +} diff --git a/common/proto/state/sqlserver/test.proto b/common/proto/state/sqlserver/test.proto new file mode 100644 index 0000000000..7c84802598 --- /dev/null +++ b/common/proto/state/sqlserver/test.proto @@ -0,0 +1,10 @@ +syntax = "proto3"; + +option go_package = "github.com/dapr/components-contrib/common/proto/state/sqlserver"; + +import "google/protobuf/timestamp.proto"; + +message TestEvent { + int32 eventId = 1; + google.protobuf.Timestamp timestamp = 2; +} \ No newline at end of file diff --git a/state/sqlserver/sqlserver.go b/state/sqlserver/sqlserver.go index 35e2f7ed20..f607d47af1 100644 --- a/state/sqlserver/sqlserver.go +++ b/state/sqlserver/sqlserver.go @@ -16,6 +16,7 @@ package sqlserver import ( "context" "database/sql" + "encoding/base64" "encoding/hex" "encoding/json" "errors" @@ -287,8 +288,15 @@ func (s *SQLServer) Get(ctx context.Context, req *state.GetRequest) (*state.GetR } } + bytes, err := base64.StdEncoding.DecodeString(data) + if err != nil { + s.logger. + WithFields(map[string]any{"error": err}). + Debug("error decoding base64 data. Fallback to []byte") + bytes = []byte(data) + } return &state.GetResponse{ - Data: []byte(data), + Data: bytes, ETag: ptr.Of(etag), Metadata: metadata, }, nil @@ -305,16 +313,23 @@ type dbExecutor interface { } func (s *SQLServer) executeSet(ctx context.Context, db dbExecutor, req *state.SetRequest) error { - var err error - var bytes []byte - bytes, err = utils.Marshal(req.Value, json.Marshal) - if err != nil { - return err + var reqValue string + + bytes, ok := req.Value.([]byte) + if !ok { + bt, err := json.Marshal(req.Value) + if err != nil { + return err + } + reqValue = string(bt) + } else { + reqValue = base64.StdEncoding.EncodeToString(bytes) } + etag := sql.Named(rowVersionColumnName, nil) if req.HasETag() { var b []byte - b, err = hex.DecodeString(*req.ETag) + b, err := hex.DecodeString(*req.ETag) if err != nil { return state.NewETagError(state.ETagInvalid, err) } @@ -327,13 +342,14 @@ func (s *SQLServer) executeSet(ctx context.Context, db dbExecutor, req *state.Se } var res sql.Result + var err error if req.Options.Concurrency == state.FirstWrite { res, err = db.ExecContext(ctx, s.upsertCommand, sql.Named(keyColumnName, req.Key), - sql.Named("Data", string(bytes)), etag, + sql.Named("Data", reqValue), etag, sql.Named("FirstWrite", 1), sql.Named("TTL", ttl)) } else { res, err = db.ExecContext(ctx, s.upsertCommand, sql.Named(keyColumnName, req.Key), - sql.Named("Data", string(bytes)), etag, + sql.Named("Data", reqValue), etag, sql.Named("FirstWrite", 0), sql.Named("TTL", ttl)) } diff --git a/state/sqlserver/sqlserver_integration_test.go b/state/sqlserver/sqlserver_integration_test.go index 7af6220476..e317128a40 100644 --- a/state/sqlserver/sqlserver_integration_test.go +++ b/state/sqlserver/sqlserver_integration_test.go @@ -30,10 +30,12 @@ import ( "testing" "time" - uuid "github.com/google/uuid" + "github.com/google/uuid" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" + "github.com/dapr/components-contrib/common/proto/state/sqlserver" "github.com/dapr/components-contrib/metadata" "github.com/dapr/components-contrib/state" "github.com/dapr/kit/logger" @@ -42,7 +44,7 @@ import ( const ( // connectionStringEnvKey defines the key containing the integration test connection string // To use docker, server=localhost;user id=sa;password=Pass@Word1;port=1433; - // To use Azure SQL, server=.database.windows.net;user id=;port=1433;password=;database=dapr_test;. + // To use Azure SQL, server=.database.windows.net;User id=;port=1433;password=;database=dapr_test;. connectionStringEnvKey = "DAPR_TEST_SQL_CONNSTRING" usersTableName = "Users" beverageTea = "tea" @@ -77,6 +79,7 @@ func TestIntegrationCases(t *testing.T) { t.Run("Multi operations", testMultiOperations) t.Run("Insert and Update Set Record Dates", testInsertAndUpdateSetRecordDates) t.Run("Multiple initializations", testMultipleInitializations) + t.Run("Should preserve byte data when not base64 encoded", testNonBase64ByteData) // Run concurrent set tests 10 times const executions = 10 @@ -112,6 +115,9 @@ func createMetadata(schema string, kt KeyType, indexedProperties string) state.M // Ensure the database is running // For docker, use: docker run --name sqlserver -e "ACCEPT_EULA=Y" -e "SA_PASSWORD=Pass@Word1" -p 1433:1433 -d mcr.microsoft.com/mssql/server:2019-GA-ubuntu-16.04. +// For azure-sql-edge use: +// docker volume create sqlvolume +// docker run --name sqlserver -e "ACCEPT_EULA=Y" -e "MSSQL_SA_PASSWORD=Pass@Word1" -e "MSSQL_PID=Developer" -e "MSSQL_AGENT_ENABLED=TRUE" -e "MSSQL_COLLATION=SQL_Latin1_General_CP1_CI_AS" -e "MSSQL_LCID=1033" -p 1433:1433 -v sqlvolume:/var/opt/mssql -d mcr.microsoft.com/azure-sql-edge:latest func getTestStore(t *testing.T, indexedProperties string) *SQLServer { return getTestStoreWithKeyType(t, StringKeyType, indexedProperties) } @@ -597,3 +603,23 @@ func testMultipleInitializations(t *testing.T) { }) } } + +func testNonBase64ByteData(t *testing.T) { + t.Run("Set And Get", func(t *testing.T) { + store := getTestStore(t, "") + request := &sqlserver.TestEvent{ + EventId: -1, + } + requestBytes, err := proto.Marshal(request) + require.NoError(t, err) + require.NoError(t, store.Set(t.Context(), &state.SetRequest{Key: "1", Value: requestBytes})) + resp, err := store.Get(t.Context(), &state.GetRequest{Key: "1"}) + require.NoError(t, err) + + response := &sqlserver.TestEvent{} + err = proto.Unmarshal(resp.Data, response) + require.NoError(t, err) + + assert.EqualValues(t, request.GetEventId(), response.GetEventId()) + }) +} From af9604275e9f4c295d384c4deac1092e219d5b0f Mon Sep 17 00:00:00 2001 From: Prashanth Nagaraj Date: Fri, 1 Aug 2025 22:25:46 +0530 Subject: [PATCH 07/30] feat: Add support for replicateSubscriptionState in Pulsar pubsub component (#3853) Signed-off-by: Prashanth Nagaraj Co-authored-by: Cassie Coyle Co-authored-by: Yaron Schneider Co-authored-by: Dapr Bot <56698301+dapr-bot@users.noreply.github.com> Signed-off-by: swatimodi-scout --- pubsub/pulsar/metadata.go | 1 + pubsub/pulsar/metadata.yaml | 13 ++++++++++- pubsub/pulsar/pulsar.go | 1 + pubsub/pulsar/pulsar_test.go | 42 ++++++++++++++++++++++++++++++++++++ 4 files changed, 56 insertions(+), 1 deletion(-) diff --git a/pubsub/pulsar/metadata.go b/pubsub/pulsar/metadata.go index b3f54c1336..c373005425 100644 --- a/pubsub/pulsar/metadata.go +++ b/pubsub/pulsar/metadata.go @@ -39,6 +39,7 @@ type pulsarMetadata struct { ReceiverQueueSize int `mapstructure:"receiverQueueSize"` SubscriptionType string `mapstructure:"subscribeType"` SubscriptionInitialPosition string `mapstructure:"subscribeInitialPosition"` + ReplicateSubscriptionState bool `mapstructure:"replicateSubscriptionState"` SubscriptionMode string `mapstructure:"subscribeMode"` Token string `mapstructure:"token"` oauth2.ClientCredentialsMetadata `mapstructure:",squash"` diff --git a/pubsub/pulsar/metadata.yaml b/pubsub/pulsar/metadata.yaml index 8277583055..72c5fd5a30 100644 --- a/pubsub/pulsar/metadata.yaml +++ b/pubsub/pulsar/metadata.yaml @@ -202,6 +202,17 @@ metadata: url: title: "Pulsar SubscriptionInitialPosition" url: "https://pkg.go.dev/github.com/apache/pulsar-client-go/pulsar#SubscriptionInitialPosition" + - name: replicateSubscriptionState + type: bool + description: | + Enable replication of subscription state across geo-replicated Pulsar clusters. + When enabled, subscription state (such as cursor positions and acknowledgments) will be replicated to other clusters in a geo-replicated setup. + This is useful for maintaining subscription consistency during cluster failovers. + default: 'false' + example: '"true", "false"' + url: + title: "Pulsar Geo-Replication" + url: "https://pulsar.apache.org/docs/administration-geo/" - name: subscribeMode type: string description: | @@ -210,4 +221,4 @@ metadata: example: '"durable"' url: title: "Pulsar SubscriptionMode" - url: "https://pkg.go.dev/github.com/apache/pulsar-client-go/pulsar#SubscriptionMode" \ No newline at end of file + url: "https://pkg.go.dev/github.com/apache/pulsar-client-go/pulsar#SubscriptionMode" diff --git a/pubsub/pulsar/pulsar.go b/pubsub/pulsar/pulsar.go index 56c1d9b322..1ce44dc209 100644 --- a/pubsub/pulsar/pulsar.go +++ b/pubsub/pulsar/pulsar.go @@ -509,6 +509,7 @@ func (p *Pulsar) Subscribe(ctx context.Context, req pubsub.SubscribeRequest, han MessageChannel: channel, NackRedeliveryDelay: p.metadata.RedeliveryDelay, ReceiverQueueSize: p.metadata.ReceiverQueueSize, + ReplicateSubscriptionState: p.metadata.ReplicateSubscriptionState, } // Handle KeySharedPolicy for key_shared subscription type diff --git a/pubsub/pulsar/pulsar_test.go b/pubsub/pulsar/pulsar_test.go index 7ade8c9fc9..eff822515b 100644 --- a/pubsub/pulsar/pulsar_test.go +++ b/pubsub/pulsar/pulsar_test.go @@ -605,6 +605,48 @@ func TestEncryptionKeys(t *testing.T) { }) } +func TestParsePulsarMetadataReplicateSubscriptionState(t *testing.T) { + tt := []struct { + name string + replicateSubscriptionState string + expected bool + }{ + { + name: "test replicateSubscriptionState true", + replicateSubscriptionState: "true", + expected: true, + }, + { + name: "test replicateSubscriptionState false", + replicateSubscriptionState: "false", + expected: false, + }, + { + name: "test replicateSubscriptionState empty (defaults to false)", + replicateSubscriptionState: "", + expected: false, + }, + } + + for _, tc := range tt { + t.Run(tc.name, func(t *testing.T) { + m := pubsub.Metadata{} + m.Properties = map[string]string{ + "host": "a", + } + + if tc.replicateSubscriptionState != "" { + m.Properties["replicateSubscriptionState"] = tc.replicateSubscriptionState + } + + meta, err := parsePulsarMetadata(m) + + require.NoError(t, err) + assert.Equal(t, tc.expected, meta.ReplicateSubscriptionState) + }) + } +} + func TestSanitiseURL(t *testing.T) { tests := []struct { name string From cc86b543eefefadca261371fd0ce892e16b11fd3 Mon Sep 17 00:00:00 2001 From: Cassie Coyle Date: Fri, 1 Aug 2025 17:16:05 -0500 Subject: [PATCH 08/30] Update PR template (#3866) Signed-off-by: Cassandra Coyle Signed-off-by: swatimodi-scout --- .github/pull_request_template.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index a21d4e8774..d5583dbf8a 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -14,4 +14,7 @@ Please make sure you've completed the relevant tasks for this PR, out of the fol * [ ] Code compiles correctly * [ ] Created/updated tests -* [ ] Extended the documentation / Created issue in the https://github.com/dapr/docs/ repo: dapr/docs#_[issue number]_ +* [ ] Extended the documentation + * [ ] Created the dapr/docs PR: + +**Note:** We expect contributors to open a corresponding documentation PR in the [dapr/docs](https://github.com/dapr/docs/) repository. As the implementer, you are the best person to document your work! Implementation PRs will not be merged until the documentation PR is opened and ready for review. From 7aa17f14d33f1c127206fa09e19193ca21f74f18 Mon Sep 17 00:00:00 2001 From: swatimodi-scout Date: Mon, 4 Aug 2025 07:41:25 +0000 Subject: [PATCH 09/30] Added enpoint in configuration Signed-off-by: swatimodi-scout --- bindings/aws/kinesis/kinesis.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/bindings/aws/kinesis/kinesis.go b/bindings/aws/kinesis/kinesis.go index bf684f8bbb..0b84b44dcd 100644 --- a/bindings/aws/kinesis/kinesis.go +++ b/bindings/aws/kinesis/kinesis.go @@ -156,8 +156,15 @@ func (a *AWSKinesis) Read(ctx context.Context, handler bindings.Handler) (err er if a.closed.Load() { return errors.New("binding is closed") } + if a.metadata.KinesisConsumerMode == SharedThroughput { - a.worker = worker.NewWorker(a.recordProcessorFactory(ctx, handler), a.authProvider.Kinesis().WorkerCfg(ctx, a.streamName, a.consumerName, a.consumerMode)) + // Configure the KCL worker with custom endpoints for LocalStack + config := a.authProvider.Kinesis().WorkerCfg(ctx, a.streamName, a.consumerName, a.consumerMode) + if a.metadata.Endpoint != "" { + config.KinesisEndpoint = a.metadata.Endpoint + config.DynamoDBEndpoint = a.metadata.Endpoint + } + a.worker = worker.NewWorker(a.recordProcessorFactory(ctx, handler), config) err = a.worker.Start() if err != nil { return err From 15d4d8810227bab52a3ac3793a69f2ca0974b65f Mon Sep 17 00:00:00 2001 From: devendrapohekar-scout Date: Tue, 5 Aug 2025 13:34:02 +0530 Subject: [PATCH 10/30] fix: Handle 'extended' KinesisConsumerMode and prevent nil pointer errors - Invoke logic only when KinesisConsumerMode is set to 'extended' to avoid unnecessary calls - Skip next step if error is not nil to prevent nil pointer errors on StreamDescription.StreamARN. Signed-off-by: swatimodi-scout --- bindings/aws/kinesis/kinesis.go | 25 +++++++++++++++++-------- common/authentication/aws/client.go | 9 +++++++-- 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/bindings/aws/kinesis/kinesis.go b/bindings/aws/kinesis/kinesis.go index 0b84b44dcd..0b269e078d 100644 --- a/bindings/aws/kinesis/kinesis.go +++ b/bindings/aws/kinesis/kinesis.go @@ -156,8 +156,8 @@ func (a *AWSKinesis) Read(ctx context.Context, handler bindings.Handler) (err er if a.closed.Load() { return errors.New("binding is closed") } - - if a.metadata.KinesisConsumerMode == SharedThroughput { + switch a.metadata.KinesisConsumerMode { + case SharedThroughput: // Configure the KCL worker with custom endpoints for LocalStack config := a.authProvider.Kinesis().WorkerCfg(ctx, a.streamName, a.consumerName, a.consumerMode) if a.metadata.Endpoint != "" { @@ -169,7 +169,7 @@ func (a *AWSKinesis) Read(ctx context.Context, handler bindings.Handler) (err er if err != nil { return err } - } else if a.metadata.KinesisConsumerMode == ExtendedFanout { + case ExtendedFanout: var stream *kinesis.DescribeStreamOutput stream, err = a.authProvider.Kinesis().Kinesis.DescribeStream(&kinesis.DescribeStreamInput{StreamName: &a.metadata.StreamName}) if err != nil { @@ -181,10 +181,18 @@ func (a *AWSKinesis) Read(ctx context.Context, handler bindings.Handler) (err er } } - stream, err := a.authProvider.Kinesis().Stream(ctx, a.streamName) - if err != nil { - return fmt.Errorf("failed to get kinesis stream arn: %v", err) + var stream *string + /** + * Invoke this only when KinesisConsumerMode is set to 'extended' to avoid unnecessary calls. + */ + if a.metadata.KinesisConsumerMode == ExtendedFanout { + streamARN, err := a.authProvider.Kinesis().Stream(ctx, a.streamName) + if err != nil { + return fmt.Errorf("failed to get kinesis stream arn: %v", err) + } + stream = streamARN } + // Wait for context cancelation then stop a.wg.Add(1) go func() { @@ -193,9 +201,10 @@ func (a *AWSKinesis) Read(ctx context.Context, handler bindings.Handler) (err er case <-ctx.Done(): case <-a.closeCh: } - if a.metadata.KinesisConsumerMode == SharedThroughput { + switch a.metadata.KinesisConsumerMode { + case SharedThroughput: a.worker.Shutdown() - } else if a.metadata.KinesisConsumerMode == ExtendedFanout { + case ExtendedFanout: a.deregisterConsumer(ctx, stream, a.consumerARN) } }() diff --git a/common/authentication/aws/client.go b/common/authentication/aws/client.go index 11b26e4988..08a22991c5 100644 --- a/common/authentication/aws/client.go +++ b/common/authentication/aws/client.go @@ -205,9 +205,14 @@ func (c *KinesisClients) Stream(ctx context.Context, streamName string) (*string stream, err := c.Kinesis.DescribeStreamWithContext(ctx, &kinesis.DescribeStreamInput{ StreamName: aws.String(streamName), }) - if stream != nil { - return stream.StreamDescription.StreamARN, err + /** + * If the error is not nil, do not proceed to the next step + * as it may cause a nil pointer error on stream.StreamDescription.StreamARN. + */ + if err != nil { + return nil, err } + return stream.StreamDescription.StreamARN, err } return nil, errors.New("unable to get stream arn due to empty client") From 2d77bbf0acf56f69989c7957b432211d1205992a Mon Sep 17 00:00:00 2001 From: devendrapohekar-scout Date: Tue, 5 Aug 2025 14:47:39 +0530 Subject: [PATCH 11/30] fix: update comments. Signed-off-by: swatimodi-scout --- bindings/aws/kinesis/kinesis.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/bindings/aws/kinesis/kinesis.go b/bindings/aws/kinesis/kinesis.go index 0b269e078d..32c7b01829 100644 --- a/bindings/aws/kinesis/kinesis.go +++ b/bindings/aws/kinesis/kinesis.go @@ -156,8 +156,7 @@ func (a *AWSKinesis) Read(ctx context.Context, handler bindings.Handler) (err er if a.closed.Load() { return errors.New("binding is closed") } - switch a.metadata.KinesisConsumerMode { - case SharedThroughput: + if a.metadata.KinesisConsumerMode == SharedThroughput { // Configure the KCL worker with custom endpoints for LocalStack config := a.authProvider.Kinesis().WorkerCfg(ctx, a.streamName, a.consumerName, a.consumerMode) if a.metadata.Endpoint != "" { @@ -169,7 +168,7 @@ func (a *AWSKinesis) Read(ctx context.Context, handler bindings.Handler) (err er if err != nil { return err } - case ExtendedFanout: + } else if a.metadata.KinesisConsumerMode == ExtendedFanout { var stream *kinesis.DescribeStreamOutput stream, err = a.authProvider.Kinesis().Kinesis.DescribeStream(&kinesis.DescribeStreamInput{StreamName: &a.metadata.StreamName}) if err != nil { @@ -201,10 +200,9 @@ func (a *AWSKinesis) Read(ctx context.Context, handler bindings.Handler) (err er case <-ctx.Done(): case <-a.closeCh: } - switch a.metadata.KinesisConsumerMode { - case SharedThroughput: + if a.metadata.KinesisConsumerMode == SharedThroughput { a.worker.Shutdown() - case ExtendedFanout: + } else if a.metadata.KinesisConsumerMode == ExtendedFanout { a.deregisterConsumer(ctx, stream, a.consumerARN) } }() From f7251b7c0468d0e1b2f6a28ff73c06fb8b8eb392 Mon Sep 17 00:00:00 2001 From: Filinto Duran <1373693+filintod@users.noreply.github.com> Date: Thu, 7 Aug 2025 09:02:04 -0500 Subject: [PATCH 12/30] Conversation Tool Calling fix on echo and other minor fixes (#3930) Signed-off-by: Filinto Duran <1373693+filintod@users.noreply.github.com> Co-authored-by: Yaron Schneider Signed-off-by: swatimodi-scout --- conversation/deepseek/deepseek.go | 4 + conversation/echo/echo.go | 113 +++++++++++----- conversation/echo/echo_test.go | 127 ++++++------------ conversation/mistral/mistral.go | 31 ++++- .../conversation/huggingface/huggingface.yml | 2 +- .../conformance/conversation/conversation.go | 67 ++++----- 6 files changed, 174 insertions(+), 170 deletions(-) diff --git a/conversation/deepseek/deepseek.go b/conversation/deepseek/deepseek.go index d0f234af1c..1ed6c075fd 100644 --- a/conversation/deepseek/deepseek.go +++ b/conversation/deepseek/deepseek.go @@ -59,6 +59,10 @@ func (d *Deepseek) Init(ctx context.Context, meta conversation.Metadata) error { model = md.Model } + if md.Endpoint == "" { + md.Endpoint = defaultEndpoint + } + options := []openai.Option{ openai.WithModel(model), openai.WithToken(md.Key), diff --git a/conversation/echo/echo.go b/conversation/echo/echo.go index bb97e567fa..43fadc67f5 100644 --- a/conversation/echo/echo.go +++ b/conversation/echo/echo.go @@ -18,6 +18,8 @@ import ( "context" "fmt" "reflect" + "strconv" + "strings" "github.com/tmc/langchaingo/llms" @@ -61,67 +63,104 @@ func (e *Echo) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { // Converse returns one output per input message. func (e *Echo) Converse(ctx context.Context, r *conversation.Request) (res *conversation.Response, err error) { - if r.Message == nil { + if r == nil || r.Message == nil { return &conversation.Response{ ConversationContext: r.ConversationContext, Outputs: []conversation.Result{}, }, nil } - outputs := make([]conversation.Result, 0, len(*r.Message)) + // if we get tools, respond with tool calls for each tool + var toolCalls []llms.ToolCall + if r.Tools != nil { + // create tool calls for each tool + toolCalls = make([]llms.ToolCall, 0, len(*r.Tools)) + for id, tool := range *r.Tools { + // extract argument names from parameters.properties + if tool.Function == nil { + continue // skip if no function + } + // try to get parameters/arg-names from tool function if any + var parameters map[string]any + var argNames []string + if tool.Function.Parameters != nil { + // ensure parameters are a map + ok := false + parameters, ok = tool.Function.Parameters.(map[string]any) + if !ok { + return nil, fmt.Errorf("tool function parameters must be a map[string]any, got %T", tool.Function.Parameters) + } + } + // try get arg names from properties + if properties, ok := parameters["properties"]; ok { + _, ok = properties.(map[string]any) + if !ok { + return nil, fmt.Errorf("tool function properties must be a map[string]any, got %T", properties) + } + if propMap, ok := properties.(map[string]any); ok && len(propMap) != 0 { + argNames = make([]string, 0, len(propMap)) + for argName := range propMap { + argNames = append(argNames, argName) + } + } + } - for _, message := range *r.Message { - var content string - var toolCalls []llms.ToolCall + toolCalls = append(toolCalls, llms.ToolCall{ + ID: strconv.Itoa(id), + Type: tool.Type, + FunctionCall: &llms.FunctionCall{ + Name: tool.Function.Name, + Arguments: strings.Join(argNames, ","), + }, + }) + } + } - for i, part := range message.Parts { + // iterate over each message in the request to echo back the content in the response. We respond with the acummulated content of the message parts and tool responses + contentFromMessaged := make([]string, 0, len(*r.Message)) + for _, message := range *r.Message { + for _, part := range message.Parts { switch p := part.(type) { case llms.TextContent: - // end with space if not the first part - if i > 0 && content != "" { - content += " " - } - content += p.Text + // append to slice that we'll join later with new line separators + contentFromMessaged = append(contentFromMessaged, p.Text) case llms.ToolCall: + // in case we added explicit tool calls on the request like on multi-turn conversations. We still append tool calls for each tool defined in the request. toolCalls = append(toolCalls, p) case llms.ToolCallResponse: - content = p.Content - toolCalls = append(toolCalls, llms.ToolCall{ - ID: p.ToolCallID, - Type: "function", - FunctionCall: &llms.FunctionCall{ - Name: p.Name, - Arguments: p.Content, - }, - }) + // show tool responses on the request like on multi-turn conversations + contentFromMessaged = append(contentFromMessaged, fmt.Sprintf("Tool Response for tool ID '%s' with name '%s': %s", p.ToolCallID, p.Name, p.Content)) default: return nil, fmt.Errorf("found invalid content type as input for %v", p) } } + } - choice := conversation.Choice{ - FinishReason: "stop", - Index: 0, - Message: conversation.Message{ - Content: content, - }, - } - - if len(toolCalls) > 0 { - choice.Message.ToolCallRequest = &toolCalls - } + stopReason := "stop" + if len(toolCalls) > 0 { + stopReason = "tool_calls" + // follows openai spec for tool_calls finish reason https://platform.openai.com/docs/api-reference/chat/object + } + choice := conversation.Choice{ + FinishReason: stopReason, + Index: 0, + Message: conversation.Message{ + Content: strings.Join(contentFromMessaged, "\n"), + }, + } - output := conversation.Result{ - StopReason: "stop", - Choices: []conversation.Choice{choice}, - } + if len(toolCalls) > 0 { + choice.Message.ToolCallRequest = &toolCalls + } - outputs = append(outputs, output) + output := conversation.Result{ + StopReason: stopReason, + Choices: []conversation.Choice{choice}, } res = &conversation.Response{ ConversationContext: r.ConversationContext, - Outputs: outputs, + Outputs: []conversation.Result{output}, } return res, nil diff --git a/conversation/echo/echo_test.go b/conversation/echo/echo_test.go index 2eed047db9..410c623eec 100644 --- a/conversation/echo/echo_test.go +++ b/conversation/echo/echo_test.go @@ -97,19 +97,7 @@ func TestConverse(t *testing.T) { FinishReason: "stop", Index: 0, Message: conversation.Message{ - Content: "first message second message", - }, - }, - }, - }, - { - StopReason: "stop", - Choices: []conversation.Choice{ - { - FinishReason: "stop", - Index: 0, - Message: conversation.Message{ - Content: "third message", + Content: "first message\nsecond message\nthird message", }, }, }, @@ -127,7 +115,7 @@ func TestConverse(t *testing.T) { Message: &tt.inputs, }) require.NoError(t, err) - assert.Len(t, r.Outputs, len(tt.expected.Outputs)) + assert.Len(t, r.Outputs, 1) assert.Equal(t, tt.expected.Outputs, r.Outputs) }) } @@ -137,20 +125,32 @@ func TestConverseAlpha2(t *testing.T) { tests := []struct { name string messages []llms.MessageContent + tools []llms.Tool expected *conversation.Response }{ { name: "tool call request", messages: []llms.MessageContent{ { - Role: llms.ChatMessageTypeAI, + Role: llms.ChatMessageTypeHuman, Parts: []llms.ContentPart{ - llms.ToolCall{ - ID: "myid", - Type: "function", - FunctionCall: &llms.FunctionCall{ - Name: "myfunc", - Arguments: `{"name": "Dapr"}`, + llms.TextContent{Text: "hello echo"}, + }, + }, + }, + tools: []llms.Tool{ + { + Type: "function", + Function: &llms.FunctionDefinition{ + Name: "myfunc", + Description: "A function that does something", + Parameters: map[string]any{ + "type": "object", + "properties": map[string]any{ + "name": map[string]any{ + "type": "string", + "description": "The name to process", + }, }, }, }, @@ -159,19 +159,20 @@ func TestConverseAlpha2(t *testing.T) { expected: &conversation.Response{ Outputs: []conversation.Result{ { - StopReason: "stop", + StopReason: "tool_calls", Choices: []conversation.Choice{ { - FinishReason: "stop", + FinishReason: "tool_calls", Index: 0, Message: conversation.Message{ + Content: "hello echo", ToolCallRequest: &[]llms.ToolCall{ { - ID: "myid", + ID: "0", // ID is auto-generated by the echo component Type: "function", FunctionCall: &llms.FunctionCall{ Name: "myfunc", - Arguments: `{"name": "Dapr"}`, + Arguments: "name", }, }, }, @@ -183,8 +184,15 @@ func TestConverseAlpha2(t *testing.T) { }, }, { - name: "tool call response", + name: "text message with tool call response", + // echo responds with the text message and tool call response appended to the message content messages: []llms.MessageContent{ + { + Role: llms.ChatMessageTypeHuman, + Parts: []llms.ContentPart{ + llms.TextContent{Text: "hello echo"}, + }, + }, { Role: llms.ChatMessageTypeTool, Parts: []llms.ContentPart{ @@ -205,62 +213,7 @@ func TestConverseAlpha2(t *testing.T) { FinishReason: "stop", Index: 0, Message: conversation.Message{ - Content: "Dapr", - ToolCallRequest: &[]llms.ToolCall{ - { - ID: "myid", - Type: "function", - FunctionCall: &llms.FunctionCall{ - Name: "myfunc", - Arguments: "Dapr", - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - { - name: "mixed content with text and tool call", - messages: []llms.MessageContent{ - { - Role: llms.ChatMessageTypeAI, - Parts: []llms.ContentPart{ - llms.TextContent{Text: "text msg"}, - llms.ToolCall{ - ID: "myid", - Type: "function", - FunctionCall: &llms.FunctionCall{ - Name: "myfunc", - Arguments: `{"name": "Dapr"}`, - }, - }, - }, - }, - }, - expected: &conversation.Response{ - Outputs: []conversation.Result{ - { - StopReason: "stop", - Choices: []conversation.Choice{ - { - FinishReason: "stop", - Index: 0, - Message: conversation.Message{ - Content: "text msg", - ToolCallRequest: &[]llms.ToolCall{ - { - ID: "myid", - Type: "function", - FunctionCall: &llms.FunctionCall{ - Name: "myfunc", - Arguments: `{"name": "Dapr"}`, - }, - }, - }, + Content: "hello echo\nTool Response for tool ID 'myid' with name 'myfunc': Dapr", }, }, }, @@ -275,9 +228,14 @@ func TestConverseAlpha2(t *testing.T) { e := NewEcho(logger.NewLogger("echo test")) e.Init(t.Context(), conversation.Metadata{}) - r, err := e.Converse(t.Context(), &conversation.Request{ + request := &conversation.Request{ Message: &tt.messages, - }) + } + if len(tt.tools) > 0 { + request.Tools = &tt.tools + } + + r, err := e.Converse(t.Context(), request) require.NoError(t, err) assert.Len(t, r.Outputs, 1) @@ -295,7 +253,6 @@ func TestConverseAlpha2(t *testing.T) { for j, toolCall := range *choice.Message.ToolCallRequest { expectedToolCall := (*expectedChoice.Message.ToolCallRequest)[j] - assert.Equal(t, expectedToolCall.ID, toolCall.ID) assert.Equal(t, expectedToolCall.Type, toolCall.Type) if expectedToolCall.FunctionCall != nil { diff --git a/conversation/mistral/mistral.go b/conversation/mistral/mistral.go index 3144525089..c319319747 100644 --- a/conversation/mistral/mistral.go +++ b/conversation/mistral/mistral.go @@ -17,6 +17,7 @@ package mistral import ( "context" "reflect" + "strings" "github.com/dapr/components-contrib/conversation" "github.com/dapr/components-contrib/conversation/langchaingokit" @@ -110,13 +111,33 @@ func CreateToolCallPart(toolCall *llms.ToolCall) llms.ContentPart { // using the human role specifically otherwise mistral will reject the tool response message. // Most LLM providers can handle tool call responses using the tool call response object; // however, mistral requires it as text in conversation history. -func CreateToolResponseMessage(response llms.ToolCallResponse) llms.MessageContent { - return llms.MessageContent{ +func CreateToolResponseMessage(responses ...llms.ContentPart) llms.MessageContent { + msg := llms.MessageContent{ Role: llms.ChatMessageTypeHuman, - Parts: []llms.ContentPart{ + } + if len(responses) == 0 { + return msg + } + var toolID, name string + + mistralContentParts := make([]string, 0, len(responses)) + for _, response := range responses { + if resp, ok := response.(llms.ToolCallResponse); ok { + if toolID == "" { + toolID = resp.ToolCallID + } + if name == "" { + name = resp.Name + } + mistralContentParts = append(mistralContentParts, resp.Content) + } + } + if len(mistralContentParts) > 0 { + msg.Parts = []llms.ContentPart{ llms.TextContent{ - Text: "Tool response [ID: " + response.ToolCallID + ", Name: " + response.Name + "]: " + response.Content, + Text: "Tool response [ID: " + toolID + ", Name: " + name + "]: " + strings.Join(mistralContentParts, "\n"), }, - }, + } } + return msg } diff --git a/tests/config/conversation/huggingface/huggingface.yml b/tests/config/conversation/huggingface/huggingface.yml index 4af48ad7c8..c4ca9f2fea 100644 --- a/tests/config/conversation/huggingface/huggingface.yml +++ b/tests/config/conversation/huggingface/huggingface.yml @@ -9,4 +9,4 @@ spec: - name: key value: "${{HUGGINGFACE_API_KEY}}" - name: model - value: "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B" \ No newline at end of file + value: "HuggingFaceTB/SmolLM3-3B" \ No newline at end of file diff --git a/tests/conformance/conversation/conversation.go b/tests/conformance/conversation/conversation.go index 99ff508ea9..4b9cfda368 100644 --- a/tests/conformance/conversation/conversation.go +++ b/tests/conformance/conversation/conversation.go @@ -46,6 +46,8 @@ func NewTestConfig(componentName string) TestConfig { } func ConformanceTests(t *testing.T, props map[string]string, conv conversation.Conversation, component string) { + providerStopReasons := []string{"stop", "end_turn", "FinishReasonStop", "tool_calls"} + t.Run("init", func(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) defer cancel() @@ -104,7 +106,7 @@ func ConformanceTests(t *testing.T, props map[string]string, conv conversation.C assert.Len(t, resp.Outputs, 1) assert.NotEmpty(t, resp.Outputs[0].Choices[0].Message.Content) // anthropic responds with end_turn but other llm providers return with stop - assert.True(t, slices.Contains([]string{"stop", "end_turn"}, resp.Outputs[0].StopReason)) + assert.True(t, slices.Contains(providerStopReasons, resp.Outputs[0].StopReason)) assert.Empty(t, resp.Outputs[0].Choices[0].Message.ToolCallRequest) }) t.Run("test system message type", func(t *testing.T) { @@ -133,20 +135,11 @@ func ConformanceTests(t *testing.T, props map[string]string, conv conversation.C resp, err := conv.Converse(ctx, req) require.NoError(t, err) - // Echo component returns one output per message, other components return one output - if component == "echo" { - assert.Len(t, resp.Outputs, 2) - // Check the last output - system message - assert.NotEmpty(t, resp.Outputs[1].Choices[0].Message.Content) - assert.True(t, slices.Contains([]string{"stop", "end_turn"}, resp.Outputs[1].StopReason)) - assert.Empty(t, resp.Outputs[1].Choices[0].Message.ToolCallRequest) - } else { - assert.Len(t, resp.Outputs, 1) - assert.NotEmpty(t, resp.Outputs[0].Choices[0].Message.Content) - // anthropic responds with end_turn but other llm providers return with stop - assert.True(t, slices.Contains([]string{"stop", "end_turn"}, resp.Outputs[0].StopReason)) - assert.Empty(t, resp.Outputs[0].Choices[0].Message.ToolCallRequest) - } + assert.Len(t, resp.Outputs, 1) + assert.NotEmpty(t, resp.Outputs[0].Choices[0].Message.Content) + // anthropic responds with end_turn but other llm providers return with stop + assert.True(t, slices.Contains(providerStopReasons, resp.Outputs[0].StopReason), resp.Outputs[0].StopReason) + assert.Empty(t, resp.Outputs[0].Choices[0].Message.ToolCallRequest) }) t.Run("test assistant message type", func(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), 25*time.Second) @@ -233,26 +226,16 @@ func ConformanceTests(t *testing.T, props map[string]string, conv conversation.C resp, err := conv.Converse(ctx, req) require.NoError(t, err) - // Echo component returns one output per message, other components return one output - if component == "echo" { - assert.Len(t, resp.Outputs, 4) - // Check the last output - human message - assert.NotEmpty(t, resp.Outputs[3].Choices[0].Message.Content) - assert.True(t, slices.Contains([]string{"stop", "end_turn"}, resp.Outputs[3].StopReason)) - // Check the tool call output - second output - if resp.Outputs[1].Choices[0].Message.ToolCallRequest != nil && len(*resp.Outputs[1].Choices[0].Message.ToolCallRequest) > 0 { - assert.NotEmpty(t, resp.Outputs[1].Choices[0].Message.ToolCallRequest) - require.JSONEq(t, `{"test": "value"}`, (*resp.Outputs[1].Choices[0].Message.ToolCallRequest)[0].FunctionCall.Arguments) - } - } else { - assert.Len(t, resp.Outputs, 1) - assert.NotEmpty(t, resp.Outputs[0].Choices[0].Message.Content) - // anthropic responds with end_turn but other llm providers return with stop - assert.True(t, slices.Contains([]string{"stop", "end_turn"}, resp.Outputs[0].StopReason)) - if resp.Outputs[0].Choices[0].Message.ToolCallRequest != nil && len(*resp.Outputs[0].Choices[0].Message.ToolCallRequest) > 0 { - assert.NotEmpty(t, resp.Outputs[0].Choices[0].Message.ToolCallRequest) - require.JSONEq(t, `{"test": "value"}`, (*resp.Outputs[0].Choices[0].Message.ToolCallRequest)[0].FunctionCall.Arguments) - } + + // We expect a single output. In the future, depending on request (so probably a different test), + // we might get more than one as shown in new API platform.openai.com/docs/api-reference/responses/object + assert.Len(t, resp.Outputs, 1) + assert.NotEmpty(t, resp.Outputs[0].Choices[0].Message.Content) + // anthropic responds with end_turn but other llm providers return with stop + assert.True(t, slices.Contains(providerStopReasons, resp.Outputs[0].StopReason)) + if resp.Outputs[0].Choices[0].Message.ToolCallRequest != nil && len(*resp.Outputs[0].Choices[0].Message.ToolCallRequest) > 0 { + assert.NotEmpty(t, resp.Outputs[0].Choices[0].Message.ToolCallRequest) + require.JSONEq(t, `{"test": "value"}`, (*resp.Outputs[0].Choices[0].Message.ToolCallRequest)[0].FunctionCall.Arguments) } }) @@ -277,13 +260,13 @@ func ConformanceTests(t *testing.T, props map[string]string, conv conversation.C assert.Len(t, resp.Outputs, 1) assert.NotEmpty(t, resp.Outputs[0].Choices[0].Message.Content) // anthropic responds with end_turn but other llm providers return with stop - assert.True(t, slices.Contains([]string{"stop", "end_turn"}, resp.Outputs[0].StopReason)) + assert.True(t, slices.Contains(providerStopReasons, resp.Outputs[0].StopReason)) if resp.Outputs[0].Choices[0].Message.ToolCallRequest != nil { assert.Empty(t, *resp.Outputs[0].Choices[0].Message.ToolCallRequest) } }) - t.Run("test tool message type - confirming active tool calling capability", func(t *testing.T) { + t.Run("test tool message type - confirming active tool calling capability (empty tool choice)", func(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), 25*time.Second) defer cancel() @@ -384,10 +367,10 @@ func ConformanceTests(t *testing.T, props map[string]string, conv conversation.C require.NoError(t, err2) assert.Len(t, resp2.Outputs, 1) assert.NotEmpty(t, resp2.Outputs[0].Choices[0].Message.Content) - assert.True(t, slices.Contains([]string{"stop", "end_turn"}, resp2.Outputs[0].StopReason)) + assert.True(t, slices.Contains(providerStopReasons, resp2.Outputs[0].StopReason)) } else { assert.NotEmpty(t, resp.Outputs[0].Choices[0].Message.Content) - assert.True(t, slices.Contains([]string{"stop", "end_turn"}, resp.Outputs[0].StopReason)) + assert.True(t, slices.Contains(providerStopReasons, resp.Outputs[0].StopReason)) } }) @@ -456,7 +439,7 @@ func ConformanceTests(t *testing.T, props map[string]string, conv conversation.C // check if we got a tool call request if found { assert.Equal(t, "retrieve_payment_status", toolCall.FunctionCall.Name) - assert.Contains(t, toolCall.FunctionCall.Arguments, "T1001") + assert.Contains(t, toolCall.FunctionCall.Arguments, "transaction_id") toolResponse := llms.ToolCallResponse{ ToolCallID: toolCall.ID, @@ -508,11 +491,11 @@ func ConformanceTests(t *testing.T, props map[string]string, conv conversation.C require.NoError(t, err) assert.Len(t, resp2.Outputs, 1) assert.NotEmpty(t, resp2.Outputs[0].Choices[0].Message.Content) - assert.True(t, slices.Contains([]string{"stop", "end_turn"}, resp2.Outputs[0].StopReason)) + assert.True(t, slices.Contains(providerStopReasons, resp2.Outputs[0].StopReason)) } else { // it is valid too if no tool call was generated assert.NotEmpty(t, resp1.Outputs[0].Choices[0].Message.Content) - assert.True(t, slices.Contains([]string{"stop", "end_turn"}, resp1.Outputs[0].StopReason)) + assert.True(t, slices.Contains(providerStopReasons, resp1.Outputs[0].StopReason)) } }) }) From 6b1dc037b017f40c0e8982ed8f02838d2bb68aa7 Mon Sep 17 00:00:00 2001 From: Andy Ladd Date: Thu, 7 Aug 2025 10:03:12 -0400 Subject: [PATCH 13/30] fix: 'ttlInSeconds' metadata key now respected when publishing to Azure Service Bus Topics. (#3942) Signed-off-by: Andy Ladd Co-authored-by: Yaron Schneider Signed-off-by: swatimodi-scout --- common/component/azure/servicebus/message.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/component/azure/servicebus/message.go b/common/component/azure/servicebus/message.go index 8a73a70555..0803392d5f 100644 --- a/common/component/azure/servicebus/message.go +++ b/common/component/azure/servicebus/message.go @@ -122,7 +122,7 @@ func addMetadataToMessage(asbMsg *azservicebus.Message, metadata map[string]stri switch k { // Common keys - case mdutils.TTLMetadataKey: + case mdutils.TTLMetadataKey, mdutils.TTLInSecondsMetadataKey: // Ignore v here and use TryGetTTL for the validation it performs ttl, ok, _ := mdutils.TryGetTTL(metadata) if ok { From 7cccdc6c6d782583daebabc0efbc9ce95b87d483 Mon Sep 17 00:00:00 2001 From: Javier Aliaga Date: Mon, 11 Aug 2025 18:07:12 +0200 Subject: [PATCH 14/30] chore: Move kafka to use aws sdk v2 (#3940) Signed-off-by: Javier Aliaga Co-authored-by: Yaron Schneider Signed-off-by: swatimodi-scout --- common/authentication/aws/aws.go | 23 ----- common/authentication/aws/client.go | 141 --------------------------- common/authentication/aws/static.go | 47 +-------- common/authentication/aws/x509.go | 38 +------- common/aws/auth/auth.go | 26 +++-- common/aws/auth/auth_static.go | 36 ++++--- common/component/kafka/aws.go | 121 +++++++++++++++++++++++ common/component/kafka/clients.go | 10 +- common/component/kafka/kafka.go | 52 ++++------ common/component/kafka/kafka_test.go | 38 ++++---- 10 files changed, 206 insertions(+), 326 deletions(-) create mode 100644 common/component/kafka/aws.go diff --git a/common/authentication/aws/aws.go b/common/authentication/aws/aws.go index 2728249afd..ef39402663 100644 --- a/common/authentication/aws/aws.go +++ b/common/authentication/aws/aws.go @@ -26,17 +26,6 @@ type EnvironmentSettings struct { Metadata map[string]string } -// TODO: Delete in Dapr 1.17 so we can move all IAM fields to use the defaults of: -// accessKey and secretKey and region as noted in the docs, and Options struct above. -type DeprecatedKafkaIAM struct { - Region string `json:"awsRegion" mapstructure:"awsRegion"` - AccessKey string `json:"awsAccessKey" mapstructure:"awsAccessKey"` - SecretKey string `json:"awsSecretKey" mapstructure:"awsSecretKey"` - SessionToken string `json:"awsSessionToken" mapstructure:"awsSessionToken"` - IamRoleArn string `json:"awsIamRoleArn" mapstructure:"awsIamRoleArn"` - StsSessionName string `json:"awsStsSessionName" mapstructure:"awsStsSessionName"` -} - type Options struct { Logger logger.Logger Properties map[string]string @@ -89,7 +78,6 @@ type Provider interface { ParameterStore() *ParameterStoreClients Kinesis() *KinesisClients Ses() *SesClients - Kafka(KafkaOptions) (*KafkaClients, error) // Postgres is an outlier to the others in the sense that we can update only it's config, // as we use a max connection time of 8 minutes. @@ -115,14 +103,3 @@ func NewEnvironmentSettings(md map[string]string) (EnvironmentSettings, error) { return es, nil } - -// Coalesce is a helper function to return the first non-empty string from the inputs -// This helps us to migrate away from the deprecated duplicate aws auth profile metadata fields in Dapr 1.17. -func Coalesce(values ...string) string { - for _, v := range values { - if v != "" { - return v - } - } - return "" -} diff --git a/common/authentication/aws/client.go b/common/authentication/aws/client.go index 08a22991c5..06d47bb3a0 100644 --- a/common/authentication/aws/client.go +++ b/common/authentication/aws/client.go @@ -16,13 +16,8 @@ package aws import ( "context" "errors" - "fmt" "sync" - "time" - "github.com/IBM/sarama" - "github.com/aws/aws-msk-iam-sasl-signer-go/signer" - aws2 "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/session" @@ -56,7 +51,6 @@ type Clients struct { ParameterStore *ParameterStoreClients kinesis *KinesisClients ses *SesClients - kafka *KafkaClients } func newClients() *Clients { @@ -85,14 +79,6 @@ func (c *Clients) refresh(session *session.Session) error { c.kinesis.New(session) case c.ses != nil: c.ses.New(session) - case c.kafka != nil: - // Note: we pass in nil for token provider - // as there are no special fields for x509 auth for it. - // Only static auth passes it in. - err := c.kafka.New(session, nil) - if err != nil { - return fmt.Errorf("failed to refresh Kafka AWS IAM Config: %w", err) - } } return nil } @@ -139,16 +125,6 @@ type SesClients struct { Ses *ses.SES } -type KafkaClients struct { - config *sarama.Config - consumerGroup *string - brokers *[]string - maxMessageBytes *int - - ConsumerGroup sarama.ConsumerGroup - Producer sarama.SyncProducer -} - func (c *S3Clients) New(session *session.Session) { refreshedS3 := s3.New(session, session.Config) c.S3 = refreshedS3 @@ -237,120 +213,3 @@ func (c *KinesisClients) WorkerCfg(ctx context.Context, stream, consumer, mode s func (c *SesClients) New(session *session.Session) { c.Ses = ses.New(session, session.Config) } - -type KafkaOptions struct { - Config *sarama.Config - ConsumerGroup string - Brokers []string - MaxMessageBytes int -} - -func initKafkaClients(opts KafkaOptions) *KafkaClients { - return &KafkaClients{ - config: opts.Config, - consumerGroup: &opts.ConsumerGroup, - brokers: &opts.Brokers, - maxMessageBytes: &opts.MaxMessageBytes, - } -} - -func (c *KafkaClients) New(session *session.Session, tokenProvider *mskTokenProvider) error { - const timeout = 10 * time.Second - creds, err := session.Config.Credentials.Get() - if err != nil { - return fmt.Errorf("failed to get credentials from session: %w", err) - } - - // fill in token provider common fields across x509 and static auth - if tokenProvider == nil { - tokenProvider = &mskTokenProvider{} - } - tokenProvider.generateTokenTimeout = timeout - tokenProvider.region = *session.Config.Region - tokenProvider.accessKey = creds.AccessKeyID - tokenProvider.secretKey = creds.SecretAccessKey - tokenProvider.sessionToken = creds.SessionToken - - c.config.Net.SASL.Enable = true - c.config.Net.SASL.Mechanism = sarama.SASLTypeOAuth - c.config.Net.SASL.TokenProvider = tokenProvider - - _, err = c.config.Net.SASL.TokenProvider.Token() - if err != nil { - return fmt.Errorf("error validating iam credentials %v", err) - } - - consumerGroup, err := sarama.NewConsumerGroup(*c.brokers, *c.consumerGroup, c.config) - if err != nil { - return err - } - c.ConsumerGroup = consumerGroup - - producer, err := c.getSyncProducer() - if err != nil { - return err - } - c.Producer = producer - - return nil -} - -// Kafka specific -type mskTokenProvider struct { - generateTokenTimeout time.Duration - accessKey string - secretKey string - sessionToken string - awsIamRoleArn string - awsStsSessionName string - region string -} - -func (m *mskTokenProvider) Token() (*sarama.AccessToken, error) { - // this function can't use the context passed on Init because that context would be cancelled right after Init - ctx, cancel := context.WithTimeout(context.Background(), m.generateTokenTimeout) - defer cancel() - - switch { - // we must first check if we are using the assume role auth profile - case m.awsIamRoleArn != "" && m.awsStsSessionName != "": - token, _, err := signer.GenerateAuthTokenFromRole(ctx, m.region, m.awsIamRoleArn, m.awsStsSessionName) - return &sarama.AccessToken{Token: token}, err - case m.accessKey != "" && m.secretKey != "": - token, _, err := signer.GenerateAuthTokenFromCredentialsProvider(ctx, m.region, aws2.CredentialsProviderFunc(func(ctx context.Context) (aws2.Credentials, error) { - return aws2.Credentials{ - AccessKeyID: m.accessKey, - SecretAccessKey: m.secretKey, - SessionToken: m.sessionToken, - }, nil - })) - return &sarama.AccessToken{Token: token}, err - - default: // load default aws creds - token, _, err := signer.GenerateAuthToken(ctx, m.region) - return &sarama.AccessToken{Token: token}, err - } -} - -func (c *KafkaClients) getSyncProducer() (sarama.SyncProducer, error) { - // Add SyncProducer specific properties to copy of base config - c.config.Producer.RequiredAcks = sarama.WaitForAll - c.config.Producer.Retry.Max = 5 - c.config.Producer.Return.Successes = true - - if *c.maxMessageBytes > 0 { - c.config.Producer.MaxMessageBytes = *c.maxMessageBytes - } - - saramaClient, err := sarama.NewClient(*c.brokers, c.config) - if err != nil { - return nil, err - } - - producer, err := sarama.NewSyncProducerFromClient(saramaClient) - if err != nil { - return nil, err - } - - return producer, nil -} diff --git a/common/authentication/aws/static.go b/common/authentication/aws/static.go index e79dee1841..3c3bf7a389 100644 --- a/common/authentication/aws/static.go +++ b/common/authentication/aws/static.go @@ -15,7 +15,6 @@ package aws import ( "context" - "errors" "fmt" "strconv" "sync" @@ -319,36 +318,6 @@ func (a *StaticAuth) getDatabaseToken(ctx context.Context, poolConfig *pgxpool.C return authenticationToken, nil } -func (a *StaticAuth) Kafka(opts KafkaOptions) (*KafkaClients, error) { - a.mu.Lock() - defer a.mu.Unlock() - - // This means we've already set the config in our New function - // to use the SASL token provider. - if a.clients.kafka != nil { - return a.clients.kafka, nil - } - - a.clients.kafka = initKafkaClients(opts) - // static auth has additional fields we need added, - // so we add those static auth specific fields here, - // and the rest of the token provider fields are added in New() - tokenProvider := mskTokenProvider{} - if a.assumeRoleARN != nil { - tokenProvider.awsIamRoleArn = *a.assumeRoleARN - } - if a.sessionName != "" { - tokenProvider.awsStsSessionName = a.sessionName - } - - err := a.clients.kafka.New(a.session, &tokenProvider) - if err != nil { - return nil, fmt.Errorf("failed to create AWS IAM Kafka config: %w", err) - } - - return a.clients.kafka, nil -} - func (a *StaticAuth) createSession() (*session.Session, error) { var awsConfig *aws.Config if a.cfg == nil { @@ -390,21 +359,7 @@ func (a *StaticAuth) createSession() (*session.Session, error) { } func (a *StaticAuth) Close() error { - a.mu.Lock() - defer a.mu.Unlock() - - errs := make([]error, 2) - if a.clients.kafka != nil { - if a.clients.kafka.Producer != nil { - errs[0] = a.clients.kafka.Producer.Close() - a.clients.kafka.Producer = nil - } - if a.clients.kafka.ConsumerGroup != nil { - errs[1] = a.clients.kafka.ConsumerGroup.Close() - a.clients.kafka.ConsumerGroup = nil - } - } - return errors.Join(errs...) + return nil } func GetConfigV2(accessKey string, secretKey string, sessionToken string, region string, endpoint string) (awsv2.Config, error) { diff --git a/common/authentication/aws/x509.go b/common/authentication/aws/x509.go index 6556ece74c..5a42ff4b41 100644 --- a/common/authentication/aws/x509.go +++ b/common/authentication/aws/x509.go @@ -138,23 +138,7 @@ func newX509(ctx context.Context, opts Options, cfg *aws.Config) (*x509, error) } func (a *x509) Close() error { - a.mu.Lock() - defer a.mu.Unlock() - close(a.closeCh) - a.wg.Wait() - - errs := make([]error, 2) - if a.clients.kafka != nil { - if a.clients.kafka.Producer != nil { - errs[0] = a.clients.kafka.Producer.Close() - a.clients.kafka.Producer = nil - } - if a.clients.kafka.ConsumerGroup != nil { - errs[1] = a.clients.kafka.ConsumerGroup.Close() - a.clients.kafka.ConsumerGroup = nil - } - } - return errors.Join(errs...) + return nil } func (a *x509) getCertPEM(ctx context.Context) error { @@ -409,26 +393,6 @@ func (a *x509) UpdatePostgres(ctx context.Context, poolConfig *pgxpool.Config) { } } -func (a *x509) Kafka(opts KafkaOptions) (*KafkaClients, error) { - a.mu.Lock() - defer a.mu.Unlock() - - // This means we've already set the config in our New function - // to use the SASL token provider. - if a.clients.kafka != nil { - return a.clients.kafka, nil - } - - a.clients.kafka = initKafkaClients(opts) - // Note: we pass in nil for token provider, - // as there are no special fields for x509 auth for it. - err := a.clients.kafka.New(a.session, nil) - if err != nil { - return nil, fmt.Errorf("failed to create AWS IAM Kafka config: %w", err) - } - return a.clients.kafka, nil -} - func (a *x509) initializeTrustAnchors() error { var ( trustAnchor arn.ARN diff --git a/common/aws/auth/auth.go b/common/aws/auth/auth.go index c33a64bc0a..7923f9eaf3 100644 --- a/common/aws/auth/auth.go +++ b/common/aws/auth/auth.go @@ -22,13 +22,14 @@ type Options struct { Logger logger.Logger Properties map[string]string - Region string `json:"region" mapstructure:"region" mapstructurealiases:"awsRegion"` - AccessKey string `json:"accessKey" mapstructure:"accessKey"` - SecretKey string `json:"secretKey" mapstructure:"secretKey"` - SessionToken string `json:"sessionToken" mapstructure:"sessionToken"` - AssumeRoleArn string `json:"assumeRoleArn" mapstructure:"assumeRoleArn"` - TrustAnchorArn string `json:"trustAnchorArn" mapstructure:"trustAnchorArn"` - TrustProfileArn string `json:"trustProfileArn" mapstructure:"trustProfileArn"` + Region string `json:"region" mapstructure:"region" mapstructurealiases:"awsRegion"` + AccessKey string `json:"accessKey" mapstructure:"accessKey"` + SecretKey string `json:"secretKey" mapstructure:"secretKey"` + SessionToken string `json:"sessionToken" mapstructure:"sessionToken"` + AssumeRoleArn string `json:"assumeRoleArn" mapstructure:"assumeRoleArn"` + AssumeRoleSessionName string `json:"assumeRoleSessionName" mapstructure:"assumeRoleSessionName"` + TrustAnchorArn string `json:"trustAnchorArn" mapstructure:"trustAnchorArn"` + TrustProfileArn string `json:"trustProfileArn" mapstructure:"trustProfileArn"` Endpoint string `json:"endpoint" mapstructure:"endpoint"` } @@ -48,3 +49,14 @@ func NewCredentialProvider(ctx context.Context, opts Options, configOpts []func( } return newAuthStatic(ctx, opts, configOpts) } + +// Coalesce is a helper function to return the first non-empty string from the inputs +// This helps us to migrate away from the deprecated duplicate aws auth profile metadata fields in Dapr 1.17. +func Coalesce(values ...string) string { + for _, v := range values { + if v != "" { + return v + } + } + return "" +} diff --git a/common/aws/auth/auth_static.go b/common/aws/auth/auth_static.go index 13f24a5435..3b5926bc13 100644 --- a/common/aws/auth/auth_static.go +++ b/common/aws/auth/auth_static.go @@ -14,14 +14,15 @@ import ( ) type Static struct { - ProviderType ProviderType - Logger logger.Logger - AccessKey string - SecretKey string - SessionToken string - Region string - Endpoint string - AssumeRoleArn string + ProviderType ProviderType + Logger logger.Logger + AccessKey string + SecretKey string + SessionToken string + Region string + Endpoint string + AssumeRoleArn string + AssumeRoleSessionName string CredentialProvider aws.CredentialsProvider } @@ -39,13 +40,14 @@ func (a *Static) Type() ProviderType { func newAuthStatic(ctx context.Context, opts Options, configOpts []func(*config.LoadOptions) error) (CredentialProvider, error) { static := &Static{ - Logger: opts.Logger, - AccessKey: opts.AccessKey, - SecretKey: opts.SecretKey, - SessionToken: opts.SessionToken, - Region: opts.Region, - Endpoint: opts.Endpoint, - AssumeRoleArn: opts.AssumeRoleArn, + Logger: opts.Logger, + AccessKey: opts.AccessKey, + SecretKey: opts.SecretKey, + SessionToken: opts.SessionToken, + Region: opts.Region, + Endpoint: opts.Endpoint, + AssumeRoleArn: opts.AssumeRoleArn, + AssumeRoleSessionName: opts.AssumeRoleSessionName, } switch { @@ -62,7 +64,9 @@ func newAuthStatic(ctx context.Context, opts Options, configOpts []func(*config. } stsSvc := sts.NewFromConfig(awsCfg) - stsProvider := stscreds.NewAssumeRoleProvider(stsSvc, static.AssumeRoleArn) + stsProvider := stscreds.NewAssumeRoleProvider(stsSvc, static.AssumeRoleArn, func(options *stscreds.AssumeRoleOptions) { + options.RoleSessionName = static.AssumeRoleSessionName + }) static.ProviderType = StaticProviderTypeAssumeRole static.CredentialProvider = stsProvider static.Logger.Debug("using AssumeRole credentials provider") diff --git a/common/component/kafka/aws.go b/common/component/kafka/aws.go new file mode 100644 index 0000000000..42cc7df01f --- /dev/null +++ b/common/component/kafka/aws.go @@ -0,0 +1,121 @@ +/* +Copyright 2025 The Dapr Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kafka + +import ( + "context" + "fmt" + "time" + + "github.com/IBM/sarama" + "github.com/aws/aws-msk-iam-sasl-signer-go/signer" + "github.com/aws/aws-sdk-go-v2/aws" +) + +type AwsClients struct { + config *sarama.Config + consumerGroup *string + brokers *[]string + maxMessageBytes *int + + ConsumerGroup sarama.ConsumerGroup + Producer sarama.SyncProducer +} +type KafkaOptions struct { + Config *sarama.Config + ConsumerGroup string + Brokers []string + MaxMessageBytes int +} + +func InitAwsClients(opts KafkaOptions) *AwsClients { + return &AwsClients{ + config: opts.Config, + consumerGroup: &opts.ConsumerGroup, + brokers: &opts.Brokers, + maxMessageBytes: &opts.MaxMessageBytes, + } +} + +func (c *AwsClients) New(cfg *aws.Config) error { + const timeout = 10 * time.Second + + tokenProvider := &mskTokenProvider{ + generateTokenTimeout: timeout, + region: cfg.Region, + credentialsProvider: cfg.Credentials, + } + + c.config.Net.SASL.Enable = true + c.config.Net.SASL.Mechanism = sarama.SASLTypeOAuth + c.config.Net.SASL.TokenProvider = tokenProvider + + _, err := c.config.Net.SASL.TokenProvider.Token() + if err != nil { + return fmt.Errorf("error validating iam credentials %v", err) + } + + consumerGroup, err := sarama.NewConsumerGroup(*c.brokers, *c.consumerGroup, c.config) + if err != nil { + return err + } + c.ConsumerGroup = consumerGroup + + producer, err := c.getSyncProducer() + if err != nil { + return err + } + c.Producer = producer + + return nil +} + +// Kafka specific +type mskTokenProvider struct { + generateTokenTimeout time.Duration + region string + credentialsProvider aws.CredentialsProvider +} + +func (m *mskTokenProvider) Token() (*sarama.AccessToken, error) { + // this function can't use the context passed on Init because that context would be cancelled right after Init + ctx, cancel := context.WithTimeout(context.Background(), m.generateTokenTimeout) + defer cancel() + + token, _, err := signer.GenerateAuthTokenFromCredentialsProvider(ctx, m.region, m.credentialsProvider) + return &sarama.AccessToken{Token: token}, err +} + +func (c *AwsClients) getSyncProducer() (sarama.SyncProducer, error) { + // Add SyncProducer specific properties to copy of base config + c.config.Producer.RequiredAcks = sarama.WaitForAll + c.config.Producer.Retry.Max = 5 + c.config.Producer.Return.Successes = true + + if *c.maxMessageBytes > 0 { + c.config.Producer.MaxMessageBytes = *c.maxMessageBytes + } + + saramaClient, err := sarama.NewClient(*c.brokers, c.config) + if err != nil { + return nil, err + } + + producer, err := sarama.NewSyncProducerFromClient(saramaClient) + if err != nil { + return nil, err + } + + return producer, nil +} diff --git a/common/component/kafka/clients.go b/common/component/kafka/clients.go index 8e8111b7b2..71cb5f2b22 100644 --- a/common/component/kafka/clients.go +++ b/common/component/kafka/clients.go @@ -4,8 +4,6 @@ import ( "fmt" "github.com/IBM/sarama" - - awsAuth "github.com/dapr/components-contrib/common/authentication/aws" ) type clients struct { @@ -23,14 +21,16 @@ func (k *Kafka) latestClients() (*clients, error) { }, nil // case 1: use aws clients with refreshable tokens in the cfg - case k.awsAuthProvider != nil: - awsKafkaOpts := awsAuth.KafkaOptions{ + case k.awsConfig != nil: + awsKafkaOpts := KafkaOptions{ Config: k.config, ConsumerGroup: k.consumerGroup, Brokers: k.brokers, MaxMessageBytes: k.maxMessageBytes, } - awsKafkaClients, err := k.awsAuthProvider.Kafka(awsKafkaOpts) + + awsKafkaClients := InitAwsClients(awsKafkaOpts) + err := awsKafkaClients.New(k.awsConfig) if err != nil { return nil, fmt.Errorf("failed to get AWS IAM Kafka clients: %w", err) } diff --git a/common/component/kafka/kafka.go b/common/component/kafka/kafka.go index 50e6d1554f..76a40c002c 100644 --- a/common/component/kafka/kafka.go +++ b/common/component/kafka/kafka.go @@ -26,10 +26,12 @@ import ( "time" "github.com/IBM/sarama" + aws2 "github.com/aws/aws-sdk-go-v2/aws" "github.com/linkedin/goavro/v2" "github.com/riferrei/srclient" - awsAuth "github.com/dapr/components-contrib/common/authentication/aws" + "github.com/dapr/components-contrib/common/aws" + awsAuth "github.com/dapr/components-contrib/common/aws/auth" "github.com/dapr/components-contrib/pubsub" "github.com/dapr/kit/logger" kitmd "github.com/dapr/kit/metadata" @@ -53,7 +55,6 @@ type Kafka struct { initialOffset int64 config *sarama.Config escapeHeaders bool - awsAuthProvider awsAuth.Provider subscribeTopics TopicHandlerConfig subscribeLock sync.Mutex @@ -84,6 +85,7 @@ type Kafka struct { consumeRetryInterval time.Duration excludeHeaderMetaRegex *regexp.Regexp + awsConfig *aws2.Config } type SchemaType int @@ -194,27 +196,17 @@ func (k *Kafka) Init(ctx context.Context, metadata map[string]string) error { // already handled in updateTLSConfig case awsIAMAuthType: k.logger.Info("Configuring AWS IAM authentication") - kafkaIAM, validateErr := k.ValidateAWS(metadata) + opts, validateErr := k.ValidateAWS(metadata) if validateErr != nil { return fmt.Errorf("failed to validate AWS IAM authentication fields: %w", validateErr) } - opts := awsAuth.Options{ - Logger: k.logger, - Properties: metadata, - Region: kafkaIAM.Region, - Endpoint: "", - AccessKey: kafkaIAM.AccessKey, - SecretKey: kafkaIAM.SecretKey, - SessionToken: kafkaIAM.SessionToken, - AssumeRoleARN: kafkaIAM.IamRoleArn, - SessionName: kafkaIAM.StsSessionName, - } - var provider awsAuth.Provider - provider, err = awsAuth.NewProvider(ctx, opts, awsAuth.GetConfig(opts)) - if err != nil { - return err + + awsConfig, configErr := aws.NewConfig(ctx, opts) + if configErr != nil { + return configErr } - k.awsAuthProvider = provider + + k.awsConfig = &awsConfig } k.config = config @@ -287,7 +279,7 @@ func (k *Kafka) initConsumerGroupRebalanceStrategy(config *sarama.Config, metada k.logger.Infof("Consumer group rebalance strategy set to '%s'", config.Consumer.Group.Rebalance.GroupStrategies[0].Name()) } -func (k *Kafka) ValidateAWS(metadata map[string]string) (*awsAuth.DeprecatedKafkaIAM, error) { +func (k *Kafka) ValidateAWS(metadata map[string]string) (awsAuth.Options, error) { const defaultSessionName = "DaprDefaultSession" // This is needed as we remove the aws prefixed fields to use the builtin AWS profile fields instead. region := awsAuth.Coalesce(metadata["region"], metadata["awsRegion"]) @@ -298,16 +290,16 @@ func (k *Kafka) ValidateAWS(metadata map[string]string) (*awsAuth.DeprecatedKafk token := awsAuth.Coalesce(metadata["sessionToken"], metadata["awsSessionToken"]) if region == "" { - return nil, errors.New("metadata property AWSRegion is missing") + return awsAuth.Options{}, errors.New("metadata property AWSRegion is missing") } - return &awsAuth.DeprecatedKafkaIAM{ - Region: region, - AccessKey: accessKey, - SecretKey: secretKey, - IamRoleArn: role, - StsSessionName: session, - SessionToken: token, + return awsAuth.Options{ + Region: region, + AccessKey: accessKey, + SecretKey: secretKey, + AssumeRoleArn: role, + AssumeRoleSessionName: session, + SessionToken: token, }, nil } @@ -340,10 +332,6 @@ func (k *Kafka) Close() error { k.clients.consumerGroup = nil } } - if k.awsAuthProvider != nil { - errs[2] = k.awsAuthProvider.Close() - k.awsAuthProvider = nil - } } return errors.Join(errs...) diff --git a/common/component/kafka/kafka_test.go b/common/component/kafka/kafka_test.go index a150f88c28..c6375d94e2 100644 --- a/common/component/kafka/kafka_test.go +++ b/common/component/kafka/kafka_test.go @@ -8,13 +8,13 @@ import ( "time" "github.com/IBM/sarama" - gomock "github.com/golang/mock/gomock" + "github.com/golang/mock/gomock" "github.com/linkedin/goavro/v2" "github.com/riferrei/srclient" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - awsAuth "github.com/dapr/components-contrib/common/authentication/aws" + awsAuth "github.com/dapr/components-contrib/common/aws/auth" mock_srclient "github.com/dapr/components-contrib/common/component/kafka/mocks" "github.com/dapr/kit/logger" ) @@ -359,7 +359,7 @@ func TestValidateAWS(t *testing.T) { tests := []struct { name string metadata map[string]string - expected *awsAuth.DeprecatedKafkaIAM + expected awsAuth.Options err error }{ { @@ -372,13 +372,13 @@ func TestValidateAWS(t *testing.T) { "sessionName": "testSessionName", "sessionToken": "testSessionToken", }, - expected: &awsAuth.DeprecatedKafkaIAM{ - Region: "us-east-1", - AccessKey: "testAccessKey", - SecretKey: "testSecretKey", - IamRoleArn: "testRoleArn", - StsSessionName: "testSessionName", - SessionToken: "testSessionToken", + expected: awsAuth.Options{ + Region: "us-east-1", + AccessKey: "testAccessKey", + SecretKey: "testSecretKey", + AssumeRoleArn: "testRoleArn", + AssumeRoleSessionName: "testSessionName", + SessionToken: "testSessionToken", }, err: nil, }, @@ -392,13 +392,13 @@ func TestValidateAWS(t *testing.T) { "awsStsSessionName": "awsSessionName", "awsSessionToken": "awsSessionToken", }, - expected: &awsAuth.DeprecatedKafkaIAM{ - Region: "us-west-2", - AccessKey: "awsAccessKey", - SecretKey: "awsSecretKey", - IamRoleArn: "awsRoleArn", - StsSessionName: "awsSessionName", - SessionToken: "awsSessionToken", + expected: awsAuth.Options{ + Region: "us-west-2", + AccessKey: "awsAccessKey", + SecretKey: "awsSecretKey", + AssumeRoleArn: "awsRoleArn", + AssumeRoleSessionName: "awsSessionName", + SessionToken: "awsSessionToken", }, err: nil, }, @@ -408,13 +408,13 @@ func TestValidateAWS(t *testing.T) { "accessKey": "key", "secretKey": "secret", }, - expected: nil, + expected: awsAuth.Options{}, err: errors.New("metadata property AWSRegion is missing"), }, { name: "Empty metadata", metadata: map[string]string{}, - expected: nil, + expected: awsAuth.Options{}, err: errors.New("metadata property AWSRegion is missing"), }, } From 0fbe3a7340c0e922033a6448f9c1b8f5664cb522 Mon Sep 17 00:00:00 2001 From: Mike Nguyen Date: Tue, 12 Aug 2025 17:04:22 +0100 Subject: [PATCH 15/30] fix(common): add missing region and remove session token requirement (#3914) Signed-off-by: Mike Nguyen Co-authored-by: Yaron Schneider Signed-off-by: swatimodi-scout --- bindings/aws/s3/s3.go | 136 +++++++++++++-------------------- common/aws/auth/auth_static.go | 2 +- common/aws/config.go | 7 ++ go.mod | 7 +- go.sum | 14 +--- tests/certification/go.mod | 8 +- tests/certification/go.sum | 16 +--- 7 files changed, 66 insertions(+), 124 deletions(-) diff --git a/bindings/aws/s3/s3.go b/bindings/aws/s3/s3.go index 0846b929d2..5b82b05192 100644 --- a/bindings/aws/s3/s3.go +++ b/bindings/aws/s3/s3.go @@ -24,20 +24,17 @@ import ( "net/http" "os" "reflect" - "slices" "strings" "time" - "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/aws/aws-sdk-go/aws" - awsCommon "github.com/dapr/components-contrib/common/aws" - awsCommonAuth "github.com/dapr/components-contrib/common/aws/auth" - - "github.com/aws/aws-sdk-go-v2/feature/s3/manager" - "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go/service/s3/s3manager" "github.com/google/uuid" "github.com/dapr/components-contrib/bindings" + awsAuth "github.com/dapr/components-contrib/common/authentication/aws" commonutils "github.com/dapr/components-contrib/common/utils" "github.com/dapr/components-contrib/metadata" "github.com/dapr/kit/logger" @@ -63,12 +60,9 @@ const ( // AWSS3 is a binding for an AWS S3 storage bucket. type AWSS3 struct { - metadata *s3Metadata - logger logger.Logger - s3Client *s3.Client - s3Uploader *manager.Uploader - s3Downloader *manager.Downloader - s3PresignClient *s3.PresignClient + metadata *s3Metadata + authProvider awsAuth.Provider + logger logger.Logger } type s3Metadata struct { @@ -112,42 +106,10 @@ func NewAWSS3(logger logger.Logger) bindings.OutputBinding { return &AWSS3{logger: logger} } -// Init does metadata parsing and connection creation. -func (s *AWSS3) Init(ctx context.Context, metadata bindings.Metadata) error { - m, err := s.parseMetadata(metadata) - if err != nil { - return err - } - s.metadata = m - - authOpts := awsCommonAuth.Options{ - Logger: s.logger, - - Properties: metadata.Properties, - - Region: m.Region, - Endpoint: m.Endpoint, - AccessKey: m.AccessKey, - SecretKey: m.SecretKey, - SessionToken: m.SessionToken, - } - - var configOptions []awsCommon.ConfigOption - - var s3Options []func(options *s3.Options) - - if s.metadata.DisableSSL { - s3Options = append(s3Options, func(options *s3.Options) { - options.EndpointOptions.DisableHTTPS = true - }) - } - - if !s.metadata.ForcePathStyle { - s3Options = append(s3Options, func(options *s3.Options) { - options.UsePathStyle = true - }) - } +func (s *AWSS3) getAWSConfig(opts awsAuth.Options) *aws.Config { + cfg := awsAuth.GetConfig(opts).WithS3ForcePathStyle(s.metadata.ForcePathStyle).WithDisableSSL(s.metadata.DisableSSL) + // Use a custom HTTP client to allow self-signed certs if s.metadata.InsecureSSL { customTransport := http.DefaultTransport.(*http.Transport).Clone() customTransport.TLSClientConfig = &tls.Config{ @@ -157,27 +119,44 @@ func (s *AWSS3) Init(ctx context.Context, metadata bindings.Metadata) error { client := &http.Client{ Transport: customTransport, } - configOptions = append(configOptions, awsCommon.WithHTTPClient(client)) + cfg = cfg.WithHTTPClient(client) s.logger.Infof("aws s3: you are using 'insecureSSL' to skip server config verify which is unsafe!") } + return cfg +} - awsConfig, err := awsCommon.NewConfig(ctx, authOpts, configOptions...) +// Init does metadata parsing and connection creation. +func (s *AWSS3) Init(ctx context.Context, metadata bindings.Metadata) error { + m, err := s.parseMetadata(metadata) if err != nil { - return fmt.Errorf("s3 binding error: failed to create AWS config: %w", err) + return err } + s.metadata = m - s.s3Client = s3.NewFromConfig(awsConfig, s3Options...) - - s.s3Uploader = manager.NewUploader(s.s3Client) - s.s3Downloader = manager.NewDownloader(s.s3Client) - - s.s3PresignClient = s3.NewPresignClient(s.s3Client) + opts := awsAuth.Options{ + Logger: s.logger, + Properties: metadata.Properties, + Region: m.Region, + Endpoint: m.Endpoint, + AccessKey: m.AccessKey, + SecretKey: m.SecretKey, + SessionToken: m.SessionToken, + } + // extra configs needed per component type + provider, err := awsAuth.NewProvider(ctx, opts, s.getAWSConfig(opts)) + if err != nil { + return err + } + s.authProvider = provider return nil } func (s *AWSS3) Close() error { + if s.authProvider != nil { + return s.authProvider.Close() + } return nil } @@ -236,25 +215,19 @@ func (s *AWSS3) create(ctx context.Context, req *bindings.InvokeRequest) (*bindi r = b64.NewDecoder(b64.StdEncoding, r) } - var storageClass types.StorageClass + var storageClass *string if metadata.StorageClass != "" { - // assert storageclass exists in the types.storageclass.values() slice - storageClass = types.StorageClass(strings.ToUpper(metadata.StorageClass)) - if !slices.Contains(storageClass.Values(), storageClass) { - return nil, fmt.Errorf("s3 binding error: invalid storage class '%s' provided", metadata.StorageClass) - } + storageClass = aws.String(metadata.StorageClass) } - s3UploaderPutObjectInput := &s3.PutObjectInput{ + resultUpload, err := s.authProvider.S3().Uploader.UploadWithContext(ctx, &s3manager.UploadInput{ Bucket: ptr.Of(metadata.Bucket), Key: ptr.Of(key), Body: r, ContentType: contentType, StorageClass: storageClass, Tagging: tagging, - } - - resultUpload, err := s.s3Uploader.Upload(ctx, s3UploaderPutObjectInput) + }) if err != nil { return nil, fmt.Errorf("s3 binding error: uploading failed: %w", err) } @@ -323,21 +296,16 @@ func (s *AWSS3) presignObject(ctx context.Context, bucket, key, ttl string) (str if err != nil { return "", fmt.Errorf("s3 binding error: cannot parse duration %s: %w", ttl, err) } - s3GetObjectInput := &s3.GetObjectInput{ + objReq, _ := s.authProvider.S3().S3.GetObjectRequest(&s3.GetObjectInput{ Bucket: ptr.Of(bucket), Key: ptr.Of(key), - } - - presignedObjectRequest, err := s.s3PresignClient.PresignGetObject( - ctx, - s3GetObjectInput, - s3.WithPresignExpires(d), - ) + }) + url, err := objReq.Presign(d) if err != nil { return "", fmt.Errorf("s3 binding error: failed to presign URL: %w", err) } - return presignedObjectRequest.URL, nil + return url, nil } func (s *AWSS3) get(ctx context.Context, req *bindings.InvokeRequest) (*bindings.InvokeResponse, error) { @@ -352,7 +320,7 @@ func (s *AWSS3) get(ctx context.Context, req *bindings.InvokeRequest) (*bindings } buff := &aws.WriteAtBuffer{} - _, err = s.s3Downloader.Download(ctx, + _, err = s.authProvider.S3().Downloader.DownloadWithContext(ctx, buff, &s3.GetObjectInput{ Bucket: ptr.Of(s.metadata.Bucket), @@ -360,8 +328,8 @@ func (s *AWSS3) get(ctx context.Context, req *bindings.InvokeRequest) (*bindings }, ) if err != nil { - var awsErr *types.NoSuchKey - if errors.As(err, &awsErr) { + var awsErr awserr.Error + if errors.As(err, &awsErr) && awsErr.Code() == s3.ErrCodeNoSuchKey { return nil, errors.New("object not found") } return nil, fmt.Errorf("s3 binding error: error downloading S3 object: %w", err) @@ -386,7 +354,7 @@ func (s *AWSS3) delete(ctx context.Context, req *bindings.InvokeRequest) (*bindi if key == "" { return nil, fmt.Errorf("s3 binding error: required metadata '%s' missing", metadataKey) } - _, err := s.s3Client.DeleteObject( + _, err := s.authProvider.S3().S3.DeleteObjectWithContext( ctx, &s3.DeleteObjectInput{ Bucket: ptr.Of(s.metadata.Bucket), @@ -394,8 +362,8 @@ func (s *AWSS3) delete(ctx context.Context, req *bindings.InvokeRequest) (*bindi }, ) if err != nil { - var awsErr *types.NoSuchKey - if errors.As(err, &awsErr) { + var awsErr awserr.Error + if errors.As(err, &awsErr) && awsErr.Code() == s3.ErrCodeNoSuchKey { return nil, errors.New("object not found") } return nil, fmt.Errorf("s3 binding error: delete operation failed: %w", err) @@ -415,9 +383,9 @@ func (s *AWSS3) list(ctx context.Context, req *bindings.InvokeRequest) (*binding if payload.MaxResults < 1 { payload.MaxResults = defaultMaxResults } - result, err := s.s3Client.ListObjects(ctx, &s3.ListObjectsInput{ + result, err := s.authProvider.S3().S3.ListObjectsWithContext(ctx, &s3.ListObjectsInput{ Bucket: ptr.Of(s.metadata.Bucket), - MaxKeys: ptr.Of(payload.MaxResults), + MaxKeys: ptr.Of(int64(payload.MaxResults)), Marker: ptr.Of(payload.Marker), Prefix: ptr.Of(payload.Prefix), Delimiter: ptr.Of(payload.Delimiter), diff --git a/common/aws/auth/auth_static.go b/common/aws/auth/auth_static.go index 3b5926bc13..ec9206ae3b 100644 --- a/common/aws/auth/auth_static.go +++ b/common/aws/auth/auth_static.go @@ -51,7 +51,7 @@ func newAuthStatic(ctx context.Context, opts Options, configOpts []func(*config. } switch { - case static.AccessKey != "" && static.SecretKey != "" && static.SessionToken != "": + case static.AccessKey != "" && static.SecretKey != "": static.ProviderType = StaticProviderTypeStatic static.CredentialProvider = credentials.NewStaticCredentialsProvider(opts.AccessKey, opts.SecretKey, opts.SessionToken) diff --git a/common/aws/config.go b/common/aws/config.go index df61f74431..7e4acdfed7 100644 --- a/common/aws/config.go +++ b/common/aws/config.go @@ -54,6 +54,13 @@ func NewConfig(ctx context.Context, authOptions auth.Options, opts ...ConfigOpti ) } + if authOptions.Region != "" { + configLoadOptions = append( + configLoadOptions, + config.WithRegion(authOptions.Region), + ) + } + if options.HTTPClient != nil { configLoadOptions = append( configLoadOptions, diff --git a/go.mod b/go.mod index 7ca8ed109b..9d61f17c01 100644 --- a/go.mod +++ b/go.mod @@ -46,10 +46,8 @@ require ( github.com/aws/aws-sdk-go-v2/credentials v1.17.70 github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.19.3 github.com/aws/aws-sdk-go-v2/feature/rds/auth v1.3.10 - github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.83 github.com/aws/aws-sdk-go-v2/service/bedrockruntime v1.17.3 github.com/aws/aws-sdk-go-v2/service/dynamodb v1.43.4 - github.com/aws/aws-sdk-go-v2/service/s3 v1.83.0 github.com/aws/aws-sdk-go-v2/service/sns v1.34.7 github.com/aws/aws-sdk-go-v2/service/sqs v1.38.8 github.com/aws/aws-sdk-go-v2/service/sts v1.34.0 @@ -201,16 +199,13 @@ require ( github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.36 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.36 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.36 // indirect github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.25.6 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.4 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.4 // indirect github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.10.17 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.17 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.17 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.25.5 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.3 // indirect - github.com/aws/smithy-go v1.22.4 // indirect + github.com/aws/smithy-go v1.22.5 // indirect github.com/awslabs/kinesis-aggregation/go v0.0.0-20210630091500-54e17340d32f // indirect github.com/benbjohnson/clock v1.3.5 // indirect github.com/beorn7/perks v1.0.1 // indirect diff --git a/go.sum b/go.sum index 8fae8aa1c8..fb3f11c723 100644 --- a/go.sum +++ b/go.sum @@ -298,8 +298,6 @@ github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.32 h1:KAXP9JSHO1vKGCr5f4O6Wm github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.32/go.mod h1:h4Sg6FQdexC1yYG9RDnOvLbW1a/P986++/Y/a+GyEM8= github.com/aws/aws-sdk-go-v2/feature/rds/auth v1.3.10 h1:z6fAXB4HSuYjrE/P8RU3NdCaN+EPaeq/+80aisCjuF8= github.com/aws/aws-sdk-go-v2/feature/rds/auth v1.3.10/go.mod h1:PoPjOi7j+/DtKIGC58HRfcdWKBPYYXwdKnRG+po+hzo= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.83 h1:08otkOELsIi0toRRGMytlJhOctcN8xfKfKFR2NXz3kE= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.83/go.mod h1:dGsGb2wI8JDWeMAhjVPP+z+dqvYjL6k6o+EujcRNk5c= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.36 h1:SsytQyTMHMDPspp+spo7XwXTP44aJZZAC7fBV2C5+5s= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.36/go.mod h1:Q1lnJArKRXkenyog6+Y+zr7WDpk4e6XlR6gs20bbeNo= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.36 h1:i2vNHQiXUvKhs3quBR6aqlgJaiaexz/aNvdCktW/kAM= @@ -307,8 +305,6 @@ github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.36/go.mod h1:UdyGa7Q91id github.com/aws/aws-sdk-go-v2/internal/ini v1.2.4/go.mod h1:ZcBrrI3zBKlhGFNYWvju0I3TR93I7YIgAfy82Fh4lcQ= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.36 h1:GMYy2EOWfzdP3wfVAGXBNKY5vK4K8vMET4sYOYltmqs= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.36/go.mod h1:gDhdAV6wL3PmPqBhiPbnlS447GoWs8HTTOYef9/9Inw= github.com/aws/aws-sdk-go-v2/service/appconfig v1.4.2/go.mod h1:FZ3HkCe+b10uFZZkFdvf98LHW21k49W8o8J366lqVKY= github.com/aws/aws-sdk-go-v2/service/bedrockruntime v1.17.3 h1:PtP2Zzf3uy94EsVOW+tB7gNt63fFZEHuS9IRWg5q250= github.com/aws/aws-sdk-go-v2/service/bedrockruntime v1.17.3/go.mod h1:4zuvYEUJm0Vq8tb3gcb2sl04A9I1AA5DKAefbYPA4VM= @@ -318,17 +314,11 @@ github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.25.6 h1:QHaS/SHXfyNycuu4 github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.25.6/go.mod h1:He/RikglWUczbkV+fkdpcV/3GdL/rTRNVy7VaUiezMo= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.4 h1:CXV68E2dNqhuynZJPB80bhPQwAKqBWVer887figW6Jc= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.4/go.mod h1:/xFi9KtvBXP97ppCz1TAEvU1Uf66qvid89rbem3wCzQ= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.4 h1:nAP2GYbfh8dd2zGZqFRSMlq+/F6cMPBUuCsGAMkN074= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.4/go.mod h1:LT10DsiGjLWh4GbjInf9LQejkYEhBgBCjLG5+lvk4EE= github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.10.17 h1:x187MqiHwBGjMGAed8Y8K1VGuCtFvQvXb24r+bwmSdo= github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.10.17/go.mod h1:mC9qMbA6e1pwEq6X3zDGtZRXMG2YaElJkbJlMVHLs5I= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.3.2/go.mod h1:72HRZDLMtmVQiLG2tLfQcaWLCssELvGl+Zf2WVxMmR8= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.17 h1:t0E6FzREdtCsiLIoLCWsYliNsRBgyGD/MCK571qk4MI= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.17/go.mod h1:ygpklyoaypuyDvOM5ujWGrYWpAK3h7ugnmKCU/76Ys4= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.17 h1:qcLWgdhq45sDM9na4cvXax9dyLitn8EYBRl8Ak4XtG4= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.17/go.mod h1:M+jkjBFZ2J6DJrjMv2+vkBbuht6kxJYtJiwoVgX4p4U= -github.com/aws/aws-sdk-go-v2/service/s3 v1.83.0 h1:5Y75q0RPQoAbieyOuGLhjV9P3txvYgXv2lg0UwJOfmE= -github.com/aws/aws-sdk-go-v2/service/s3 v1.83.0/go.mod h1:kUklwasNoCn5YpyAqC/97r6dzTA1SRKJfKq16SXeoDU= github.com/aws/aws-sdk-go-v2/service/sns v1.34.7 h1:OBuZE9Wt8h2imuRktu+WfjiTGrnYdCIJg8IX92aalHE= github.com/aws/aws-sdk-go-v2/service/sns v1.34.7/go.mod h1:4WYoZAhHt+dWYpoOQUgkUKfuQbE6Gg/hW4oXE0pKS9U= github.com/aws/aws-sdk-go-v2/service/sqs v1.38.8 h1:80dpSqWMwx2dAm30Ib7J6ucz1ZHfiv5OCRwN/EnCOXQ= @@ -344,8 +334,8 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.34.0/go.mod h1:7ph2tGpfQvwzgistp2+zg github.com/aws/rolesanywhere-credential-helper v1.0.4 h1:kHIVVdyQQiFZoKBP+zywBdFilGCS8It+UvW5LolKbW8= github.com/aws/rolesanywhere-credential-helper v1.0.4/go.mod h1:QVGNxlDlYhjR0/ZUee7uGl0hNChWidNpe2+GD87Buqk= github.com/aws/smithy-go v1.8.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= -github.com/aws/smithy-go v1.22.4 h1:uqXzVZNuNexwc/xrh6Tb56u89WDlJY6HS+KC0S4QSjw= -github.com/aws/smithy-go v1.22.4/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI= +github.com/aws/smithy-go v1.22.5 h1:P9ATCXPMb2mPjYBgueqJNCA5S9UfktsW0tTxi+a7eqw= +github.com/aws/smithy-go v1.22.5/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI= github.com/awslabs/kinesis-aggregation/go v0.0.0-20210630091500-54e17340d32f h1:Pf0BjJDga7C98f0vhw+Ip5EaiE07S3lTKpIYPNS0nMo= github.com/awslabs/kinesis-aggregation/go v0.0.0-20210630091500-54e17340d32f/go.mod h1:SghidfnxvX7ribW6nHI7T+IBbc9puZ9kk5Tx/88h8P4= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= diff --git a/tests/certification/go.mod b/tests/certification/go.mod index 63c126379a..756f93f666 100644 --- a/tests/certification/go.mod +++ b/tests/certification/go.mod @@ -85,32 +85,26 @@ require ( github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect github.com/aws/aws-msk-iam-sasl-signer-go v1.0.1-0.20241125194140-078c08b8574a // indirect github.com/aws/aws-sdk-go-v2 v1.36.5 // indirect - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.11 // indirect github.com/aws/aws-sdk-go-v2/config v1.29.17 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.17.70 // indirect github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.19.3 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.32 // indirect github.com/aws/aws-sdk-go-v2/feature/rds/auth v1.3.10 // indirect - github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.83 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.36 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.36 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.36 // indirect github.com/aws/aws-sdk-go-v2/service/dynamodb v1.43.4 // indirect github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.25.6 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.4 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.4 // indirect github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.10.17 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.17 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.17 // indirect - github.com/aws/aws-sdk-go-v2/service/s3 v1.83.0 // indirect github.com/aws/aws-sdk-go-v2/service/sns v1.34.7 // indirect github.com/aws/aws-sdk-go-v2/service/sqs v1.38.8 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.25.5 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.3 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.34.0 // indirect github.com/aws/rolesanywhere-credential-helper v1.0.4 // indirect - github.com/aws/smithy-go v1.22.4 // indirect + github.com/aws/smithy-go v1.22.5 // indirect github.com/benbjohnson/clock v1.3.5 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bits-and-blooms/bitset v1.4.0 // indirect diff --git a/tests/certification/go.sum b/tests/certification/go.sum index 13fe7f6236..1c384a2b81 100644 --- a/tests/certification/go.sum +++ b/tests/certification/go.sum @@ -209,8 +209,6 @@ github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZw github.com/aws/aws-sdk-go-v2 v1.9.2/go.mod h1:cK/D0BBs0b/oWPIcX/Z/obahJK1TT7IPVjy53i/mX/4= github.com/aws/aws-sdk-go-v2 v1.36.5 h1:0OF9RiEMEdDdZEMqF9MRjevyxAQcf6gY+E7vwBILFj0= github.com/aws/aws-sdk-go-v2 v1.36.5/go.mod h1:EYrzvCCN9CMUTa5+6lf6MM4tq3Zjp8UhSGR/cBsjai0= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.11 h1:12SpdwU8Djs+YGklkinSSlcrPyj3H4VifVsKf78KbwA= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.11/go.mod h1:dd+Lkp6YmMryke+qxW/VnKyhMBDTYP41Q2Bb+6gNZgY= github.com/aws/aws-sdk-go-v2/config v1.8.3/go.mod h1:4AEiLtAb8kLs7vgw2ZV3p2VZ1+hBavOc84hqxVNpCyw= github.com/aws/aws-sdk-go-v2/config v1.29.17 h1:jSuiQ5jEe4SAMH6lLRMY9OVC+TqJLP5655pBGjmnjr0= github.com/aws/aws-sdk-go-v2/config v1.29.17/go.mod h1:9P4wwACpbeXs9Pm9w1QTh6BwWwJjwYvJ1iCt5QbCXh8= @@ -224,8 +222,6 @@ github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.32 h1:KAXP9JSHO1vKGCr5f4O6Wm github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.32/go.mod h1:h4Sg6FQdexC1yYG9RDnOvLbW1a/P986++/Y/a+GyEM8= github.com/aws/aws-sdk-go-v2/feature/rds/auth v1.3.10 h1:z6fAXB4HSuYjrE/P8RU3NdCaN+EPaeq/+80aisCjuF8= github.com/aws/aws-sdk-go-v2/feature/rds/auth v1.3.10/go.mod h1:PoPjOi7j+/DtKIGC58HRfcdWKBPYYXwdKnRG+po+hzo= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.83 h1:08otkOELsIi0toRRGMytlJhOctcN8xfKfKFR2NXz3kE= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.83/go.mod h1:dGsGb2wI8JDWeMAhjVPP+z+dqvYjL6k6o+EujcRNk5c= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.36 h1:SsytQyTMHMDPspp+spo7XwXTP44aJZZAC7fBV2C5+5s= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.36/go.mod h1:Q1lnJArKRXkenyog6+Y+zr7WDpk4e6XlR6gs20bbeNo= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.36 h1:i2vNHQiXUvKhs3quBR6aqlgJaiaexz/aNvdCktW/kAM= @@ -233,8 +229,6 @@ github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.36/go.mod h1:UdyGa7Q91id github.com/aws/aws-sdk-go-v2/internal/ini v1.2.4/go.mod h1:ZcBrrI3zBKlhGFNYWvju0I3TR93I7YIgAfy82Fh4lcQ= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.36 h1:GMYy2EOWfzdP3wfVAGXBNKY5vK4K8vMET4sYOYltmqs= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.36/go.mod h1:gDhdAV6wL3PmPqBhiPbnlS447GoWs8HTTOYef9/9Inw= github.com/aws/aws-sdk-go-v2/service/appconfig v1.4.2/go.mod h1:FZ3HkCe+b10uFZZkFdvf98LHW21k49W8o8J366lqVKY= github.com/aws/aws-sdk-go-v2/service/dynamodb v1.43.4 h1:Rv6o9v2AfdEIKoAa7pQpJ5ch9ji2HevFUvGY6ufawlI= github.com/aws/aws-sdk-go-v2/service/dynamodb v1.43.4/go.mod h1:mWB0GE1bqcVSvpW7OtFA0sKuHk52+IqtnsYU2jUfYAs= @@ -242,17 +236,11 @@ github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.25.6 h1:QHaS/SHXfyNycuu4 github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.25.6/go.mod h1:He/RikglWUczbkV+fkdpcV/3GdL/rTRNVy7VaUiezMo= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.4 h1:CXV68E2dNqhuynZJPB80bhPQwAKqBWVer887figW6Jc= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.4/go.mod h1:/xFi9KtvBXP97ppCz1TAEvU1Uf66qvid89rbem3wCzQ= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.4 h1:nAP2GYbfh8dd2zGZqFRSMlq+/F6cMPBUuCsGAMkN074= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.4/go.mod h1:LT10DsiGjLWh4GbjInf9LQejkYEhBgBCjLG5+lvk4EE= github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.10.17 h1:x187MqiHwBGjMGAed8Y8K1VGuCtFvQvXb24r+bwmSdo= github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.10.17/go.mod h1:mC9qMbA6e1pwEq6X3zDGtZRXMG2YaElJkbJlMVHLs5I= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.3.2/go.mod h1:72HRZDLMtmVQiLG2tLfQcaWLCssELvGl+Zf2WVxMmR8= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.17 h1:t0E6FzREdtCsiLIoLCWsYliNsRBgyGD/MCK571qk4MI= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.17/go.mod h1:ygpklyoaypuyDvOM5ujWGrYWpAK3h7ugnmKCU/76Ys4= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.17 h1:qcLWgdhq45sDM9na4cvXax9dyLitn8EYBRl8Ak4XtG4= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.17/go.mod h1:M+jkjBFZ2J6DJrjMv2+vkBbuht6kxJYtJiwoVgX4p4U= -github.com/aws/aws-sdk-go-v2/service/s3 v1.83.0 h1:5Y75q0RPQoAbieyOuGLhjV9P3txvYgXv2lg0UwJOfmE= -github.com/aws/aws-sdk-go-v2/service/s3 v1.83.0/go.mod h1:kUklwasNoCn5YpyAqC/97r6dzTA1SRKJfKq16SXeoDU= github.com/aws/aws-sdk-go-v2/service/sns v1.34.7 h1:OBuZE9Wt8h2imuRktu+WfjiTGrnYdCIJg8IX92aalHE= github.com/aws/aws-sdk-go-v2/service/sns v1.34.7/go.mod h1:4WYoZAhHt+dWYpoOQUgkUKfuQbE6Gg/hW4oXE0pKS9U= github.com/aws/aws-sdk-go-v2/service/sqs v1.38.8 h1:80dpSqWMwx2dAm30Ib7J6ucz1ZHfiv5OCRwN/EnCOXQ= @@ -268,8 +256,8 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.34.0/go.mod h1:7ph2tGpfQvwzgistp2+zg github.com/aws/rolesanywhere-credential-helper v1.0.4 h1:kHIVVdyQQiFZoKBP+zywBdFilGCS8It+UvW5LolKbW8= github.com/aws/rolesanywhere-credential-helper v1.0.4/go.mod h1:QVGNxlDlYhjR0/ZUee7uGl0hNChWidNpe2+GD87Buqk= github.com/aws/smithy-go v1.8.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= -github.com/aws/smithy-go v1.22.4 h1:uqXzVZNuNexwc/xrh6Tb56u89WDlJY6HS+KC0S4QSjw= -github.com/aws/smithy-go v1.22.4/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI= +github.com/aws/smithy-go v1.22.5 h1:P9ATCXPMb2mPjYBgueqJNCA5S9UfktsW0tTxi+a7eqw= +github.com/aws/smithy-go v1.22.5/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o= From fff8e4cc3cb2ffcef78ba83588989fb4bfac2457 Mon Sep 17 00:00:00 2001 From: Sam Date: Tue, 12 Aug 2025 11:37:35 -0500 Subject: [PATCH 16/30] feat: add pubsub proper registrations (#3957) Signed-off-by: Samantha Coyle Signed-off-by: swatimodi-scout --- pubsub/jetstream/metadata.yaml | 196 ++++++++++++++++++++++++++++++++ pubsub/kubemq/metadata.yaml | 47 ++++++++ pubsub/rocketmq/metadata.go | 3 + pubsub/rocketmq/metadata.yaml | 202 +++++++++++++++++++++++++++++++++ pubsub/rocketmq/rocketmq.go | 2 + 5 files changed, 450 insertions(+) create mode 100644 pubsub/jetstream/metadata.yaml create mode 100644 pubsub/kubemq/metadata.yaml create mode 100644 pubsub/rocketmq/metadata.yaml diff --git a/pubsub/jetstream/metadata.yaml b/pubsub/jetstream/metadata.yaml new file mode 100644 index 0000000000..0124f6a525 --- /dev/null +++ b/pubsub/jetstream/metadata.yaml @@ -0,0 +1,196 @@ +# yaml-language-server: $schema=../../../component-metadata-schema.json +schemaVersion: v1 +type: pubsub +name: jetstream +version: v1 +status: beta +title: "JetStream" +urls: + - title: Reference + url: https://docs.dapr.io/reference/components-reference/supported-pubsub/setup-jetstream/ +authenticationProfiles: + - title: "Basic Authentication" + description: "Connect to NATS JetStream without authentication." + metadata: + - name: natsURL + type: string + required: true + description: The NATS server URL. + example: "nats://localhost:4222" + - title: "Token Authentication" + description: "Connect to NATS JetStream using token authentication." + metadata: + - name: natsURL + type: string + required: true + description: The NATS server URL. + example: "nats://localhost:4222" + - name: token + type: string + required: true + description: The NATS authentication token. + example: "auth-token" + - title: "JWT Authentication" + description: "Connect to NATS JetStream using JWT authentication." + metadata: + - name: natsURL + type: string + required: true + description: The NATS server URL. + example: "nats://localhost:4222" + - name: jwt + type: string + required: true + description: The JWT token for authentication. + example: "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9..." + - name: seedKey + type: string + required: true + description: The seed key for JWT authentication. + example: "SUAELX7XZIRRCNOQKCXQWFN6M3EBKUTBWK6YQKL4QFLKCUQJSEZL7ZCL7KQ" + - title: "TLS Authentication" + description: "Connect to NATS JetStream using TLS client certificates." + metadata: + - name: natsURL + type: string + required: true + description: The NATS server URL. + example: "nats://localhost:4222" + - name: tls_client_cert + type: string + required: true + description: The TLS client certificate. + example: | + -----BEGIN CERTIFICATE----- + XXX + -----END CERTIFICATE----- + - name: tls_client_key + type: string + required: true + description: The TLS client private key. + example: | + -----BEGIN PRIVATE KEY----- + XXX + -----END PRIVATE KEY----- +metadata: + - name: name + type: string + required: false + description: The name of the JetStream connection. + example: "dapr-jetstream" + default: "dapr.io - pubsub.jetstream" + - name: streamName + type: string + required: false + description: The name of the JetStream stream. + example: "my-stream" + - name: durableName + type: string + required: false + description: The durable name for the consumer. + example: "my-durable" + - name: queueGroupName + type: string + required: false + description: The queue group name for load balancing. + example: "my-queue-group" + - name: startSequence + type: integer + required: false + description: The starting sequence number for message delivery. + example: 1 + - name: startTime + type: integer + required: false + description: The starting time (Unix timestamp) for message delivery. + example: 1640995200 + default: 0 + - name: flowControl + type: bool + required: false + description: Enable flow control for the consumer. + example: false + default: false + - name: ackWait + type: string + required: false + description: The acknowledgment wait time. + example: "30s" + - name: maxDeliver + type: integer + required: false + description: The maximum number of message deliveries. + example: 5 + - name: maxAckPending + type: integer + required: false + description: The maximum number of unacknowledged messages. + example: 100 + - name: replicas + type: integer + required: false + description: The number of stream replicas. + example: 3 + - name: memoryStorage + type: bool + required: false + description: Use memory storage for the stream. + example: false + default: false + - name: rateLimit + type: integer + required: false + description: The rate limit for message consumption. + example: 1000 + - name: heartbeat + type: string + required: false + description: The heartbeat interval. + example: "30s" + - name: deliverPolicy + type: string + required: true + description: The delivery policy (all, last, new, sequence, time). + example: "all" + allowedValues: + - "all" + - "last" + - "new" + - "sequence" + - "time" + - name: ackPolicy + type: string + required: false + description: The acknowledgment policy. + example: "explicit" + allowedValues: + - "explicit" + - "all" + - "none" + default: "explicit" + - name: domain + type: string + required: false + description: The JetStream domain. + example: "my-domain" + - name: apiPrefix + type: string + required: false + description: The API prefix for JetStream. + example: "$JS.API" + - name: concurrency + type: string + required: false + description: The concurrency mode (single, parallel). + example: "single" + default: "single" + - name: backOff + type: array + required: false + description: The backoff configuration for message delivery for the consumer. + example: "[1s, 2s, 4s]" + - name: maxAckPending + type: integer + required: false + description: The maximum number of unacknowledged messages for the consumer. + example: 100 diff --git a/pubsub/kubemq/metadata.yaml b/pubsub/kubemq/metadata.yaml new file mode 100644 index 0000000000..b4acab0ff7 --- /dev/null +++ b/pubsub/kubemq/metadata.yaml @@ -0,0 +1,47 @@ +# yaml-language-server: $schema=../../../component-metadata-schema.json +schemaVersion: v1 +type: pubsub +name: kubemq +version: v1 +status: beta +title: "KubeMQ" +urls: + - title: Reference + url: https://docs.dapr.io/reference/components-reference/supported-pubsub/setup-kubemq/ +authenticationProfiles: + - title: "Token Authentication" + description: "Connect to KubeMQ using an authentication token." + metadata: + - name: authToken + type: string + required: true + description: The authentication token for KubeMQ. + example: "auth-token" +metadata: + - name: address + type: string + required: true + description: The KubeMQ server address in format host:port. + example: "localhost:50000" + - name: clientID + type: string + required: false + description: The client ID for the KubeMQ connection. + example: "dapr-client" + - name: group + type: string + required: false + description: The consumer group name. + example: "my-group" + - name: store + type: bool + required: false + description: Whether to use KubeMQ Event Store (true) or Events (false). + example: true + default: true + - name: disableReDelivery + type: bool + required: false + description: Disable message re-delivery on error. + example: false + default: false diff --git a/pubsub/rocketmq/metadata.go b/pubsub/rocketmq/metadata.go index a26d561bba..4817e01b9f 100644 --- a/pubsub/rocketmq/metadata.go +++ b/pubsub/rocketmq/metadata.go @@ -42,6 +42,9 @@ const ( DaprQueueSelector QueueSelectorType = "dapr" ) +// TODO: the time fields in the metadata need to be standardized on either seconds or milliseconds or minutes. +// Right now, it's a mix of all three. + // RocketMQ Go Client Options type rocketMQMetaData struct { // rocketmq instance name, it will be registered to the broker diff --git a/pubsub/rocketmq/metadata.yaml b/pubsub/rocketmq/metadata.yaml new file mode 100644 index 0000000000..3bf37eb6bd --- /dev/null +++ b/pubsub/rocketmq/metadata.yaml @@ -0,0 +1,202 @@ +# yaml-language-server: $schema=../../../component-metadata-schema.json +schemaVersion: v1 +type: pubsub +name: rocketmq +version: v1 +status: alpha +title: "RocketMQ" +urls: + - title: Reference + url: https://docs.dapr.io/reference/components-reference/supported-pubsub/setup-rocketmq/ +authenticationProfiles: + - title: "Credential Authentication" + description: "Connect to RocketMQ using access key and secret key." + metadata: + - name: accessKey + type: string + required: true + description: The RocketMQ access key. + example: "access-key" + - name: secretKey + type: string + required: true + description: The RocketMQ secret key. + example: "secret-key" + - name: securityToken + type: string + required: false + description: The RocketMQ security token. + example: "security-token" +metadata: + - name: nameServer + type: string + required: false + description: The RocketMQ name server address. + example: "localhost:9876" + - name: instanceName + type: string + required: false + description: The RocketMQ instance name. + example: "dapr-rocketmq" + - name: producerGroup + type: string + required: false + description: The producer group name. + example: "dapr-producer-group" + - name: consumerGroup + type: string + required: false + description: The consumer group name. + example: "dapr-consumer-group" + - name: nameSpace + type: string + required: false + description: The RocketMQ namespace. + example: "my-namespace" + - name: nameServerDomain + type: string + required: false + description: The RocketMQ name server domain. + example: "rocketmq.example.com" + - name: retries + type: number + required: false + description: The number of retry attempts for sending messages. + example: 3 + default: 3 + - name: producerQueueSelector + type: string + required: false + description: The producer queue selector type. + example: "hash" + allowedValues: + - "hash" + - "random" + - "manual" + - "roundRobin" + - "dapr" + - name: consumerModel + type: string + required: false + description: The consumer model (clustering, broadcasting). + example: "clustering" + default: "clustering" + - name: fromWhere + type: string + required: false + description: The consumption starting point. + example: "ConsumeFromLastOffset" + allowedValues: + - "ConsumeFromLastOffset" + - "ConsumeFromFirstOffset" + - "ConsumeFromTimestamp" + - name: consumeTimestamp + type: string + required: false + description: The timestamp for consumption starting point. + example: "20220817101902" + - name: consumeOrderly + type: bool + required: false + description: Enable orderly message consumption. + example: false + default: false + - name: consumeMessageBatchMaxSize + type: number + required: false + description: The maximum batch size for message consumption. + example: 10 + - name: consumeConcurrentlyMaxSpan + type: number + required: false + description: The maximum span for concurrent consumption. + example: 10 + - name: maxReconsumeTimes + type: number + required: false + description: The maximum number of reconsume times. -1 means 16 times. + example: 10000 + - name: autoCommit + type: bool + required: false + description: Enable automatic commit. + example: true + default: false + - name: consumeTimeout + type: number + required: false + description: The consume timeout in minutes. + example: 10 + - name: consumerPullTimeout + type: number + required: false + description: The consumer pull timeout in milliseconds. + example: 30 + default: 30 + - name: pullInterval + type: number + required: false + description: The pull interval in minutes. + example: 100 + default: 100 + - name: consumerBatchSize + type: number + required: false + description: The consumer batch size. + example: 10 + - name: pullBatchSize + type: number + required: false + description: The pull batch size. + example: 10 + - name: pullThresholdForQueue + type: number + required: false + description: The pull threshold for queue. + example: 100 + - name: pullThresholdForTopic + type: number + required: false + description: The pull threshold for topic. + example: 100 + - name: pullThresholdSizeForQueue + type: number + required: false + description: The pull threshold size for queue. + example: 10 + - name: pullThresholdSizeForTopic + type: number + required: false + description: The pull threshold size for topic. + example: 10 + - name: content-type + type: string + required: false + description: The content type for messages. + example: "json" + - name: sendTimeOutSec + type: number + required: false + description: The send timeout in seconds. + example: 10 + - name: logLevel + type: string + required: false + description: The log level. + example: "warn" + default: "warn" + - name: mspProperties + type: string + required: false + description: The message properties to pass to the application (comma-separated). + example: "UNIQ_KEY,MSG_ID" + - name: groupName + type: string + required: false + description: The consumer group name (deprecated, use consumerGroup instead). + example: "dapr-consumer-group" + - name: sendTimeOut + type: number + required: false + description: The send timeout in nanoseconds (deprecated, use sendTimeOutSec instead). + example: 10000000000 diff --git a/pubsub/rocketmq/rocketmq.go b/pubsub/rocketmq/rocketmq.go index c7fcdece99..d1051a9ba6 100644 --- a/pubsub/rocketmq/rocketmq.go +++ b/pubsub/rocketmq/rocketmq.go @@ -121,6 +121,7 @@ func parseNameServer(nameServer string) []string { } } +// TODO: these metadata fields need to be reevaluated on required vs not. func (r *rocketMQ) setUpConsumer() (mq.PushConsumer, error) { opts := make([]mqc.Option, 0) if r.metadata.InstanceName != "" { @@ -243,6 +244,7 @@ func (r *rocketMQ) setUpConsumer() (mq.PushConsumer, error) { return c, e } +// TODO: these metadata fields need to be reevaluated on required vs not. func (r *rocketMQ) setUpProducer() (mq.Producer, error) { opts := make([]mqp.Option, 0) if r.metadata.InstanceName != "" { From 4dd3809cecc42627f083da6ec37f0163a5a6144e Mon Sep 17 00:00:00 2001 From: Sam Date: Tue, 12 Aug 2025 11:38:46 -0500 Subject: [PATCH 17/30] feat: crypto comp registration fixes (#3954) Signed-off-by: Samantha Coyle Signed-off-by: swatimodi-scout --- crypto/azure/keyvault/metadata.yaml | 26 ++++++++++++++++++ crypto/jwks/metadata.yaml | 36 +++++++++++++++++++++++++ crypto/kubernetes/secrets/metadata.yaml | 25 +++++++++++++++++ crypto/localstorage/metadata.yaml | 18 +++++++++++++ 4 files changed, 105 insertions(+) create mode 100644 crypto/azure/keyvault/metadata.yaml create mode 100644 crypto/jwks/metadata.yaml create mode 100644 crypto/kubernetes/secrets/metadata.yaml create mode 100644 crypto/localstorage/metadata.yaml diff --git a/crypto/azure/keyvault/metadata.yaml b/crypto/azure/keyvault/metadata.yaml new file mode 100644 index 0000000000..703e3afcd3 --- /dev/null +++ b/crypto/azure/keyvault/metadata.yaml @@ -0,0 +1,26 @@ +# yaml-language-server: $schema=../../../component-metadata-schema.json +schemaVersion: v1 +type: cryptography +name: azure.keyvault +version: v1 +status: alpha +title: "Azure Key Vault" +urls: + - title: Reference + url: https://docs.dapr.io/reference/components-reference/supported-cryptography/azure-key-vault/ +builtinAuthenticationProfiles: + - name: "azuread" +metadata: + - name: vaultName + type: string + required: true + description: | + Name of the Azure Key Vault resource. + example: "key-vault" + - name: requestTimeout + type: duration + required: false + description: | + Timeout for network requests, as a Go duration string. + example: "30s" + default: "30s" diff --git a/crypto/jwks/metadata.yaml b/crypto/jwks/metadata.yaml new file mode 100644 index 0000000000..c0975723ff --- /dev/null +++ b/crypto/jwks/metadata.yaml @@ -0,0 +1,36 @@ +# yaml-language-server: $schema=../../../component-metadata-schema.json +schemaVersion: v1 +type: cryptography +name: jwks +version: v1 +status: alpha +title: "JWKS" +urls: + - title: Reference + url: https://docs.dapr.io/reference/components-reference/supported-cryptography/json-web-key-sets/ +metadata: + - name: jwks + type: string + required: true + description: | + The JWKS to use. Can be one of: + - The actual JWKS as a JSON-encoded string (optionally encoded with Base64-standard). + - A URL to a HTTP(S) endpoint returning the JWKS. + - A path to a local file containing the JWKS. + example: "https://example.com/.well-known/jwks.json" + - name: requestTimeout + type: duration + required: false + description: | + Timeout for network requests, as a Go duration string. + example: "30s" + default: "30s" + - name: minRefreshInterval + type: duration + required: false + description: | + Minimum interval before the JWKS is refreshed, as a Go duration string. + Only applies when the JWKS is fetched from a HTTP(S) URL. + example: "10m" + default: "10m" + diff --git a/crypto/kubernetes/secrets/metadata.yaml b/crypto/kubernetes/secrets/metadata.yaml new file mode 100644 index 0000000000..e0ebe11b5f --- /dev/null +++ b/crypto/kubernetes/secrets/metadata.yaml @@ -0,0 +1,25 @@ +# yaml-language-server: $schema=../../../component-metadata-schema.json +schemaVersion: v1 +type: cryptography +name: kubernetes.secrets +version: v1 +status: alpha +title: "Kubernetes Secrets" +urls: + - title: Reference + url: https://docs.dapr.io/reference/components-reference/supported-cryptography/kubernetes-secrets/ +metadata: + - name: defaultNamespace + type: string + required: false + description: | + Default namespace to retrieve secrets from. + If unset, the namespace must be specified for each key, as `namespace/secretName/key`. + example: "default" + - name: kubeconfigPath + type: string + required: false + description: | + Path to a kubeconfig file. + If empty, uses the default values. + example: "/path/to/kubeconfig" diff --git a/crypto/localstorage/metadata.yaml b/crypto/localstorage/metadata.yaml new file mode 100644 index 0000000000..1a037f543a --- /dev/null +++ b/crypto/localstorage/metadata.yaml @@ -0,0 +1,18 @@ +# yaml-language-server: $schema=../../../component-metadata-schema.json +schemaVersion: v1 +type: cryptography +name: localstorage +version: v1 +status: alpha +title: "Local Storage" +urls: + - title: Reference + url: https://docs.dapr.io/reference/components-reference/supported-cryptography/local-storage/ +metadata: + - name: path + type: string + required: true + description: | + Path to a local folder where keys are stored. + Keys are loaded from PEM or JSON (each containing an individual JWK) files from this folder. + example: "/path/to/keys" From 413c04ccc29176dd606a0df3866b2862f814244f Mon Sep 17 00:00:00 2001 From: Sam Date: Tue, 12 Aug 2025 11:42:45 -0500 Subject: [PATCH 18/30] feat: proper secretstore registration fixes (#3952) Signed-off-by: Samantha Coyle Signed-off-by: swatimodi-scout --- .../alicloud/parameterstore/metadata.yaml | 45 +++++++++++++++++++ secretstores/huaweicloud/csms/csms.go | 10 ++--- secretstores/huaweicloud/csms/metadata.yaml | 35 +++++++++++++++ secretstores/local/file/filestore.go | 6 +-- secretstores/local/file/metadata.yaml | 29 ++++++++++++ secretstores/tencentcloud/ssm/metadata.yaml | 40 +++++++++++++++++ secretstores/tencentcloud/ssm/ssm.go | 9 ++-- 7 files changed, 162 insertions(+), 12 deletions(-) create mode 100644 secretstores/alicloud/parameterstore/metadata.yaml create mode 100644 secretstores/huaweicloud/csms/metadata.yaml create mode 100644 secretstores/local/file/metadata.yaml create mode 100644 secretstores/tencentcloud/ssm/metadata.yaml diff --git a/secretstores/alicloud/parameterstore/metadata.yaml b/secretstores/alicloud/parameterstore/metadata.yaml new file mode 100644 index 0000000000..de4ffaccd8 --- /dev/null +++ b/secretstores/alicloud/parameterstore/metadata.yaml @@ -0,0 +1,45 @@ +# yaml-language-server: $schema=../../../component-metadata-schema.json +schemaVersion: v1 +type: secretstores +name: alicloud.parameterstore +version: v1 +status: alpha +title: "AliCloud OSS Parameter Store" +urls: + - title: Reference + url: https://docs.dapr.io/reference/components-reference/supported-secret-stores/alicloud-oss-parameter-store/ +authenticationProfiles: + - title: "Access Key Authentication" + description: "Authenticate using AliCloud access key and secret." + metadata: + - name: regionId + type: string + required: true + description: The AliCloud region ID. + example: "cn-hangzhou" + - name: accessKeyId + type: string + required: true + description: The AliCloud access key ID. + example: "access-key-id" + - name: accessKeySecret + type: string + required: true + description: The AliCloud access key secret. + example: "access-key-secret" + - name: securityToken + type: string + required: false + description: The AliCloud security token for temporary credentials. + example: "security-token" +metadata: + - name: version_id + type: string + required: false + description: The version ID of the parameter to retrieve. If not specified, the latest version is used. + example: "1" + - name: path + type: string + required: false + description: The path prefix for bulk operations. If not specified, root path (/) is used. + example: "/myapp/" diff --git a/secretstores/huaweicloud/csms/csms.go b/secretstores/huaweicloud/csms/csms.go index a0046b5f84..e318e4a5ea 100644 --- a/secretstores/huaweicloud/csms/csms.go +++ b/secretstores/huaweicloud/csms/csms.go @@ -48,9 +48,9 @@ type csmsSecretStore struct { } type CsmsSecretStoreMetadata struct { - Region string - AccessKey string - SecretAccessKey string + Region string `json:"region"` + AccessKey string `json:"accessKey"` + SecretAccessKey string `json:"secretAccessKey"` } // NewHuaweiCsmsSecretStore returns a new Huawei csms secret store. @@ -114,7 +114,7 @@ func (c *csmsSecretStore) BulkGetSecret(ctx context.Context, req secretstores.Bu secret, err := c.GetSecret(ctx, secretstores.GetSecretRequest{ Name: secretName, Metadata: map[string]string{ - versionID: latestVersion, + versionID: latestVersion, // TODO: make this configurable }, }) if err != nil { @@ -130,7 +130,7 @@ func (c *csmsSecretStore) BulkGetSecret(ctx context.Context, req secretstores.Bu // Get all secret names recursively. func (c *csmsSecretStore) getSecretNames(ctx context.Context, marker *string) ([]string, error) { request := &model.ListSecretsRequest{} - limit := pageLimit + limit := pageLimit // TODO: make this configurable request.Limit = &limit request.Marker = marker diff --git a/secretstores/huaweicloud/csms/metadata.yaml b/secretstores/huaweicloud/csms/metadata.yaml new file mode 100644 index 0000000000..9a925f209e --- /dev/null +++ b/secretstores/huaweicloud/csms/metadata.yaml @@ -0,0 +1,35 @@ +# yaml-language-server: $schema=../../../component-metadata-schema.json +schemaVersion: v1 +type: secretstores +name: huaweicloud.csms +version: v1 +status: alpha +title: "HuaweiCloud CSMS" +urls: + - title: Reference + url: https://docs.dapr.io/reference/components-reference/supported-secret-stores/huaweicloud-csms/ +authenticationProfiles: + - title: "Access Key Authentication" + description: "Authenticate using HuaweiCloud access key and secret." + metadata: + - name: region + type: string + required: true + description: The HuaweiCloud region. + example: "cn-north-4" + - name: accessKey + type: string + required: true + description: The HuaweiCloud access key. + example: "access-key" + - name: secretAccessKey + type: string + required: true + description: The HuaweiCloud secret access key. + example: "secret-access-key" +metadata: + - name: version_id + type: string + required: false + description: The version ID of the secret to retrieve. If not specified, the latest version is used. + example: "1" diff --git a/secretstores/local/file/filestore.go b/secretstores/local/file/filestore.go index 27bc71cfbe..a75717a655 100644 --- a/secretstores/local/file/filestore.go +++ b/secretstores/local/file/filestore.go @@ -31,9 +31,9 @@ import ( ) type localSecretStoreMetaData struct { - SecretsFile string - NestedSeparator string - MultiValued bool + SecretsFile string `json:"secretsFile"` + NestedSeparator string `json:"nestedSeparator"` + MultiValued bool `json:"multiValued"` } var _ secretstores.SecretStore = (*localSecretStore)(nil) diff --git a/secretstores/local/file/metadata.yaml b/secretstores/local/file/metadata.yaml new file mode 100644 index 0000000000..04f364551b --- /dev/null +++ b/secretstores/local/file/metadata.yaml @@ -0,0 +1,29 @@ +# yaml-language-server: $schema=../../../component-metadata-schema.json +schemaVersion: v1 +type: secretstores +name: local.file +version: v1 +status: stable +title: "Local File Secret Store" +description: "Read secrets from a local JSON file for local development." +urls: + - title: Reference + url: https://docs.dapr.io/reference/components-reference/supported-secret-stores/file-secret-store/ +metadata: + - name: secretsFile + type: string + required: true + description: Path to the JSON file containing secrets. + example: "secrets.json" + - name: nestedSeparator + type: string + required: false + description: Separator used for nested keys in the JSON file. + example: ":" + default: ":" + - name: multiValued + type: bool + required: false + description: If true, enables multiple key-values per secret feature. + example: false + default: false diff --git a/secretstores/tencentcloud/ssm/metadata.yaml b/secretstores/tencentcloud/ssm/metadata.yaml new file mode 100644 index 0000000000..4af4292aa7 --- /dev/null +++ b/secretstores/tencentcloud/ssm/metadata.yaml @@ -0,0 +1,40 @@ +# yaml-language-server: $schema=../../../component-metadata-schema.json +schemaVersion: v1 +type: secretstores +name: tencentcloud.ssm +version: v1 +status: alpha +title: "TencentCloud Secret Manager" +urls: + - title: Reference + url: https://docs.dapr.io/reference/components-reference/supported-secret-stores/ +authenticationProfiles: + - title: "Secret Key Authentication" + description: "Authenticate using TencentCloud secret ID and key." + metadata: + - name: secretId + type: string + required: true + description: The TencentCloud secret ID. + example: "secret-id" + - name: secretKey + type: string + required: true + description: The TencentCloud secret key. + example: "secret-key" + - name: token + type: string + required: false + description: The TencentCloud temporary token for temporary credentials. + example: "token" + - name: region + type: string + required: true + description: The TencentCloud region. + example: "ap-guangzhou" +metadata: + - name: VersionID + type: string + required: false + description: The version ID of the secret to retrieve. + example: "1" diff --git a/secretstores/tencentcloud/ssm/ssm.go b/secretstores/tencentcloud/ssm/ssm.go index 192d195f67..42de9bb48b 100644 --- a/secretstores/tencentcloud/ssm/ssm.go +++ b/secretstores/tencentcloud/ssm/ssm.go @@ -30,6 +30,7 @@ import ( ) const ( + // TODO: lowercase these and add to metadata struct eventually VersionID = "VersionID" RequestID = "RequestID" ValueType = "SecretValueType" @@ -56,10 +57,10 @@ type ssmSecretStore struct { } type SsmMetadata struct { - SecretID string - SecretKey string - Token string - Region string + SecretID string `json:"secretId"` + SecretKey string `json:"secretKey"` + Token string `json:"token"` + Region string `json:"region"` } // NewSSM returns a new TencentCloud ssm secret store. From b3c3b3439fdc5ac108b85d05c366052a29120c26 Mon Sep 17 00:00:00 2001 From: Sam Date: Tue, 12 Aug 2025 11:43:22 -0500 Subject: [PATCH 19/30] feat: lock comp registration fixes (#3953) Signed-off-by: Samantha Coyle Signed-off-by: swatimodi-scout --- lock/redis/metadata.yaml | 178 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 178 insertions(+) create mode 100644 lock/redis/metadata.yaml diff --git a/lock/redis/metadata.yaml b/lock/redis/metadata.yaml new file mode 100644 index 0000000000..0d633a0f3c --- /dev/null +++ b/lock/redis/metadata.yaml @@ -0,0 +1,178 @@ +# yaml-language-server: $schema=../../component-metadata-schema.json +schemaVersion: v1 +type: lock +name: redis +version: v1 +status: alpha +title: "Redis Distributed Lock" +urls: + - title: Reference + url: https://docs.dapr.io/reference/components-reference/supported-locks/redis-lock/ +lock: + operations: + - name: tryLock + description: "Attempt to acquire a distributed lock" + - name: unlock + description: "Release a distributed lock" +authenticationProfiles: + - title: "Password Authentication" + description: | + Authenticate using Redis password. + metadata: + - name: redisUsername + required: false + description: "The Redis username" + example: "redis-user" + - name: redisPassword + required: false + sensitive: true + description: "The Redis password" + example: "redis-password" + - title: "Sentinel Authentication" + description: | + Authenticate using Redis Sentinel password. + metadata: + - name: sentinelUsername + required: false + description: "The Redis Sentinel username" + example: "sentinel-user" + - name: sentinelPassword + required: false + sensitive: true + description: "The Redis Sentinel password" + example: "sentinel-password" + - title: "TLS Authentication" + description: | + Authenticate using Redis TLS certificate. + metadata: + - name: clientCert + required: false + sensitive: true + description: "The Redis client certificate" + example: '"-----BEGIN CERTIFICATE-----\nXXX..."' + - name: clientKey + required: false + sensitive: true + description: "The Redis client key" + example: '"-----BEGIN PRIVATE KEY-----\nXXX..."' +metadata: + - name: redisHost + required: true + description: "The Redis host address" + example: '"localhost:6379"' + - name: redisType + required: false + description: "The Redis type" + example: "node" + default: "node" + allowedValues: + - "node" + - "cluster" + - "sentinel" + - name: redisDB + required: false + description: "The Redis database number" + example: '0' + default: '0' + - name: redisMaxRetries + required: false + description: "Maximum Redis retries" + example: '3' + default: '3' + - name: redisMinRetryInterval + required: false + description: "Minimum Redis retry interval" + example: "8ms" + default: "8ms" + - name: redisMaxRetryInterval + required: false + description: "Maximum Redis retry interval" + example: "512ms" + default: "512ms" + - name: dialTimeout + required: false + description: "Dial timeout duration" + example: "5s" + default: "5s" + - name: readTimeout + required: false + description: "Read timeout duration" + example: "3s" + default: "3s" + - name: writeTimeout + required: false + description: "Write timeout duration" + example: "3s" + default: "3s" + - name: poolSize + required: false + description: "Connection pool size" + example: '10' + default: '10' + - name: poolTimeout + required: false + description: "Connection pool timeout" + example: "4s" + default: "4s" + - name: maxConnAge + required: false + description: "Maximum connection age" + example: "30m" + default: "30m" + - name: minIdleConns + required: false + description: "Minimum idle connections" + example: '0' + default: '0' + - name: idleTimeout + required: false + description: "Idle timeout duration" + example: "5m" + default: "5m" + - name: idleCheckFrequency + required: false + description: "Idle check frequency" + example: "1m" + default: "1m" + - name: maxRetries + required: false + description: "Maximum number of retries when attempting to acquire a lock" + example: '3' + default: '3' + - name: maxRetryBackoff + required: false + description: "Maximum backoff duration between retries" + example: "2s" + default: "2s" + - name: redeliverInterval + required: false + description: "Redeliver interval for re-attempting lock acquisition" + example: "15s" + default: "15s" + - name: processingTimeout + required: false + description: "Processing timeout for lock ownership" + example: "60s" + default: "60s" + - name: enableTLS + required: false + type: bool + description: "Whether to enable TLS encryption" + example: false + default: false + - name: useEntraID + required: false + type: bool + description: "Whether to use Entra ID for authentication" + example: false + default: false + - name: failover + required: false + type: bool + description: "Whether to enable failover mode (for Sentinel)" + example: false + default: false + - name: sentinelMasterName + required: false + description: "The Sentinel master name (used if redisType is 'sentinel')" + example: "mymaster" From 9d2a54e4c4591b9dfaa38600e611e1cd143aa7dc Mon Sep 17 00:00:00 2001 From: Sam Date: Tue, 12 Aug 2025 11:51:16 -0500 Subject: [PATCH 20/30] feat: add middleware comp proper registrations (#3956) Signed-off-by: Samantha Coyle Signed-off-by: swatimodi-scout --- middleware/http/bearer/metadata.yaml | 29 ++++++ middleware/http/oauth2/metadata.yaml | 66 ++++++++++++++ .../oauth2clientcredentials/metadata.yaml | 64 +++++++++++++ middleware/http/opa/metadata.yaml | 34 +++++++ middleware/http/ratelimit/metadata.yaml | 19 ++++ middleware/http/routerchecker/metadata.yaml | 18 ++++ middleware/http/sentinel/metadata.yaml | 90 +++++++++++++++++++ middleware/http/wasm/metadata.yaml | 27 ++++++ 8 files changed, 347 insertions(+) create mode 100644 middleware/http/bearer/metadata.yaml create mode 100644 middleware/http/oauth2/metadata.yaml create mode 100644 middleware/http/oauth2clientcredentials/metadata.yaml create mode 100644 middleware/http/opa/metadata.yaml create mode 100644 middleware/http/ratelimit/metadata.yaml create mode 100644 middleware/http/routerchecker/metadata.yaml create mode 100644 middleware/http/sentinel/metadata.yaml create mode 100644 middleware/http/wasm/metadata.yaml diff --git a/middleware/http/bearer/metadata.yaml b/middleware/http/bearer/metadata.yaml new file mode 100644 index 0000000000..1e0ee4e45d --- /dev/null +++ b/middleware/http/bearer/metadata.yaml @@ -0,0 +1,29 @@ +# yaml-language-server: $schema=../../../component-metadata-schema.json +schemaVersion: v1 +type: middleware +name: bearer +version: v1 +status: stable +title: "Bearer Token Authentication" +description: | + The Bearer middleware provides JWT token authentication for HTTP requests. + It validates Bearer tokens in the Authorization header and can extract claims for downstream processing. +urls: + - title: Reference + url: https://docs.dapr.io/reference/components-reference/supported-middleware/middleware-bearer/ +metadata: + - name: jwksURL + type: string + required: true + description: "The URL of the JSON Web Key Set (JWKS) endpoint" + example: "https://accounts.google.com/.well-known/jwks.json" + - name: issuer + type: string + required: true + description: "The expected issuer of the JWT tokens" + example: "https://accounts.google.com" + - name: audience + type: string + required: true + description: "The expected audience of the JWT tokens" + example: "my-app" diff --git a/middleware/http/oauth2/metadata.yaml b/middleware/http/oauth2/metadata.yaml new file mode 100644 index 0000000000..44ab15e727 --- /dev/null +++ b/middleware/http/oauth2/metadata.yaml @@ -0,0 +1,66 @@ +# yaml-language-server: $schema=../../../component-metadata-schema.json +schemaVersion: v1 +type: middleware +name: oauth2 +version: v1 +status: alpha +title: "OAuth2 Authentication" +description: | + The OAuth2 middleware provides OAuth2 authentication for HTTP requests. + It handles OAuth2 flows and token validation for securing API endpoints. +urls: + - title: Reference + url: https://docs.dapr.io/reference/components-reference/supported-middleware/middleware-oauth2/ +authenticationProfiles: + - title: "OAuth2 Authentication" + description: "Configure OAuth2 authentication with any OAuth2 provider" + metadata: + - name: clientID + type: string + required: true + description: "The OAuth2 client ID from your OAuth2 provider" + example: "client-id" + - name: clientSecret + type: string + required: true + description: "The OAuth2 client secret from your OAuth2 provider" + sensitive: true + example: "client-secret" + - name: authURL + type: string + required: true + description: "The OAuth2 authorization URL from your provider" + example: "https://accounts.google.com/o/oauth2/v2/auth" + - name: tokenURL + type: string + required: true + description: "The OAuth2 token URL from your provider" + example: "https://oauth2.googleapis.com/token" + - name: scopes + type: string + required: false + description: "OAuth2 scopes to request from your provider" + example: "openid profile email" +metadata: + - name: redirectURL + type: string + required: false + description: "The OAuth2 redirect URL for your application" + example: "http://localhost:8080/callback" + - name: authHeaderName + type: string + required: false + description: "The name of the authorization header to use" + example: "Authorization" + default: "Authorization" + - name: forceHTTPS + type: string + required: false + description: "Whether to force HTTPS for the redirect URL" + example: "true" + default: "false" + - name: pathFilter + type: string + required: false + description: "Regular expression to filter which paths require authentication" + example: "^/api/.*" diff --git a/middleware/http/oauth2clientcredentials/metadata.yaml b/middleware/http/oauth2clientcredentials/metadata.yaml new file mode 100644 index 0000000000..7420d1a8d0 --- /dev/null +++ b/middleware/http/oauth2clientcredentials/metadata.yaml @@ -0,0 +1,64 @@ +# yaml-language-server: $schema=../../../component-metadata-schema.json +schemaVersion: v1 +type: middleware +name: oauth2clientcredentials +version: v1 +status: alpha +title: "OAuth2 Client Credentials" +description: | + The OAuth2 Client Credentials middleware provides OAuth2 client credentials flow authentication. + It handles machine-to-machine authentication using client credentials. +urls: + - title: Reference + url: https://docs.dapr.io/reference/components-reference/supported-middleware/oauth2clientcredentials/ +authenticationProfiles: + - title: "OAuth2 Client Credentials" + description: "Configure OAuth2 client credentials authentication with any OAuth2 provider" + metadata: + - name: clientID + type: string + required: true + description: "The client ID of your application that is created as part of a credential hosted by a OAuth-enabled platform" + example: "client-id" + - name: clientSecret + type: string + required: true + description: "The client secret of your application that is created as part of a credential hosted by a OAuth-enabled platform" + sensitive: true + example: "client-secret" + - name: scopes + type: string + required: false + description: "A list of space-delimited, case-sensitive strings of scopes which are typically used for authorization in the application" + example: "https://www.googleapis.com/auth/userinfo.email" + - name: tokenURL + type: string + required: true + description: "The endpoint is used by the client to obtain an access token by presenting its authorization grant or refresh token" + example: "https://accounts.google.com/o/oauth2/token" +metadata: + - name: pathFilter + type: string + required: false + description: "Regular expression to filter which paths require authentication" + example: "^/api/.*" + - name: headerName + type: string + required: true + description: "The authorization header name to forward to your application" + example: "authorization" + - name: endpointParamsQuery + type: string + required: false + description: "Specifies additional parameters for requests to the token endpoint" + example: "param1=value1¶m2=value2" + - name: authStyle + type: integer + required: false + description: "Optionally specifies how the endpoint wants the client ID & client secret sent. 0: Auto-detect (tries both ways and caches the successful way), 1: Sends client_id and client_secret in POST body as application/x-www-form-urlencoded parameters, 2: Sends client_id and client_secret using HTTP Basic Authorization" + example: 0 + default: 0 + allowedValues: + - 0 + - 1 + - 2 diff --git a/middleware/http/opa/metadata.yaml b/middleware/http/opa/metadata.yaml new file mode 100644 index 0000000000..30747ca0bf --- /dev/null +++ b/middleware/http/opa/metadata.yaml @@ -0,0 +1,34 @@ +# yaml-language-server: $schema=../../../component-metadata-schema.json +schemaVersion: v1 +type: middleware +name: opa +version: v1 +status: alpha +title: "Open Policy Agent (OPA)" +description: | + The OPA middleware allows you to enforce policies on HTTP requests using Open Policy Agent (OPA) Rego policies. + It evaluates incoming requests against your Rego policies and can allow, deny, or modify requests based on the policy results. +urls: + - title: Reference + url: https://docs.dapr.io/reference/components-reference/supported-middleware/middleware-opa/ +metadata: + - name: rego + type: string + required: true + description: "The Rego policy code that will be evaluated for each request. The policy package must be http and the policy must set data.http.allow" + - name: defaultStatus + type: number + required: false + description: "The status code to return for denied responses" + example: 403 + default: 403 + - name: includedHeaders + type: string + required: false + description: "Comma-separated set of case-insensitive headers to include in the request input. Request headers are not passed to the policy by default. Include to receive incoming request headers in the input" + example: "x-my-custom-header, x-jwt-header" + - name: readBody + type: string + required: false + description: "Controls whether the middleware reads the entire request body in-memory and make it available for policy decisions" + example: false diff --git a/middleware/http/ratelimit/metadata.yaml b/middleware/http/ratelimit/metadata.yaml new file mode 100644 index 0000000000..95498eeb95 --- /dev/null +++ b/middleware/http/ratelimit/metadata.yaml @@ -0,0 +1,19 @@ +# yaml-language-server: $schema=../../../component-metadata-schema.json +schemaVersion: v1 +type: middleware +name: ratelimit +version: v1 +status: stable +title: "Rate Limiting" +description: | + The Rate Limiting middleware provides request rate limiting functionality. + It can limit requests based on various criteria like IP address, user, or custom keys. +urls: + - title: Reference + url: https://docs.dapr.io/reference/components-reference/supported-middleware/middleware-rate-limit/ +metadata: + - name: maxRequestsPerSecond + type: integer + required: true + description: "Maximum number of requests allowed per second" + example: 100 diff --git a/middleware/http/routerchecker/metadata.yaml b/middleware/http/routerchecker/metadata.yaml new file mode 100644 index 0000000000..d7ba54ff43 --- /dev/null +++ b/middleware/http/routerchecker/metadata.yaml @@ -0,0 +1,18 @@ +# yaml-language-server: $schema=../../../component-metadata-schema.json +schemaVersion: v1 +type: middleware +name: routerchecker +version: v1 +status: alpha +title: "Router Checker" +description: | + The RouterChecker HTTP middleware component leverages regexp to check the validity of HTTP request routing to prevent invalid routers from entering the Dapr cluster. In turn, the RouterChecker component filters out bad requests and reduces noise in the telemetry and log data. +urls: + - title: Reference + url: https://docs.dapr.io/reference/components-reference/supported-middleware/middleware-routerchecker/ +metadata: + - name: rule + type: string + required: true + description: "Regular expression to filter which paths are allowed" + example: "^[A-Za-z0-9/._-]+$" diff --git a/middleware/http/sentinel/metadata.yaml b/middleware/http/sentinel/metadata.yaml new file mode 100644 index 0000000000..c106454f45 --- /dev/null +++ b/middleware/http/sentinel/metadata.yaml @@ -0,0 +1,90 @@ +# yaml-language-server: $schema=../../../component-metadata-schema.json +schemaVersion: v1 +type: middleware +name: sentinel +version: v1 +status: alpha +title: "Sentinel" +description: | + Use Sentinel middleware to guarantee the reliability and resiliency of your application. + Sentinel is a powerful fault-tolerance component that takes "flow" as the breakthrough point and covers multiple fields including flow control, traffic shaping, concurrency limiting, circuit breaking, and adaptive system protection to guarantee the reliability and resiliency of microservices. + + The Sentinel HTTP middleware enables Dapr to facilitate Sentinel's powerful abilities to protect your application. You can refer to Sentinel Wiki for more details on Sentinel. +urls: + - title: Reference + url: https://docs.dapr.io/reference/components-reference/supported-middleware/middleware-sentinel/ +metadata: + - name: appName + type: string + required: true + description: "The application name for Sentinel" + example: "nodeapp" + - name: logDir + type: string + required: false + description: "Directory for Sentinel log files" + example: "/var/tmp" + - name: flowRules + type: string + required: false + description: "JSON array of flow control rules to limit request rate" + example: | + [ + { + "resource": "POST:/v1.0/invoke/nodeapp/method/neworder", + "threshold": 10, + "tokenCalculateStrategy": 0, + "controlBehavior": 0 + } + ] + - name: circuitBreakerRules + type: string + required: false + description: "JSON array of circuit breaker rules to handle failures" + example: | + [ + { + "resource": "POST:/v1.0/invoke/nodeapp/method/neworder", + "minRequestAmount": 5, + "statIntervalMs": 1000, + "maxAllowedRtMs": 50, + "maxSlowRequestRatio": 0.5 + } + ] + - name: hotSpotParamRules + type: string + required: false + description: "JSON array of hotspot parameter rules for parameter-based flow control" + example: | + [ + { + "resource": "POST:/v1.0/invoke/nodeapp/method/neworder", + "paramIdx": 0, + "threshold": 10, + "maxQueueingTimeMs": 500 + } + ] + - name: isolationRules + type: string + required: false + description: "JSON array of isolation rules for thread pool isolation" + example: | + [ + { + "resource": "POST:/v1.0/invoke/nodeapp/method/neworder", + "maxConcurrency": 10, + "maxQueueingTimeMs": 500 + } + ] + - name: systemRules + type: string + required: false + description: "JSON array of system protection rules for overall system protection" + example: | + [ + { + "avgRt": 50, + "maxThread": 10, + "qps": 20 + } + ] diff --git a/middleware/http/wasm/metadata.yaml b/middleware/http/wasm/metadata.yaml new file mode 100644 index 0000000000..26ae9fb586 --- /dev/null +++ b/middleware/http/wasm/metadata.yaml @@ -0,0 +1,27 @@ +# yaml-language-server: $schema=../../../component-metadata-schema.json +schemaVersion: v1 +type: middleware +name: wasm +version: v1 +status: alpha +title: "WebAssembly (WASM)" +description: | + The WebAssembly middleware allows you to run custom logic written in WASM. + It can execute WASM modules to process HTTP requests and responses. +urls: + - title: Reference + url: https://docs.dapr.io/reference/components-reference/supported-middleware/middleware-wasm/ +metadata: + - name: url + type: string + required: true + description: "URL of the WASM module" + example: "https://example.com/middleware.wasm" + - name: guestConfig + required: false + description: "Configuration object passed to the WASM module" + example: | + { + "timeout": "5s", + "maxMemory": "10MB" + } From 750a42a0423ff62ed32103627cbb582628990ab3 Mon Sep 17 00:00:00 2001 From: Nelson Parente Date: Wed, 13 Aug 2025 21:19:08 +0100 Subject: [PATCH 21/30] Revert "chore: Use base64 to store sqlserver state data" (#3966) Signed-off-by: swatimodi-scout --- common/proto/state/sqlserver/test.pb.go | 162 ------------------ common/proto/state/sqlserver/test.proto | 10 -- state/sqlserver/sqlserver.go | 34 +--- state/sqlserver/sqlserver_integration_test.go | 30 +--- 4 files changed, 11 insertions(+), 225 deletions(-) delete mode 100644 common/proto/state/sqlserver/test.pb.go delete mode 100644 common/proto/state/sqlserver/test.proto diff --git a/common/proto/state/sqlserver/test.pb.go b/common/proto/state/sqlserver/test.pb.go deleted file mode 100644 index 1c936adb57..0000000000 --- a/common/proto/state/sqlserver/test.pb.go +++ /dev/null @@ -1,162 +0,0 @@ -// Code generated by protoc-gen-go. DO NOT EDIT. -// versions: -// protoc-gen-go v1.32.0 -// protoc v4.25.4 -// source: test.proto - -package sqlserver - -import ( - "reflect" - "sync" - - "google.golang.org/protobuf/reflect/protoreflect" - "google.golang.org/protobuf/runtime/protoimpl" - "google.golang.org/protobuf/types/known/timestamppb" -) - -const ( - // Verify that this generated code is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) - // Verify that runtime/protoimpl is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) -) - -type TestEvent struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - EventId int32 `protobuf:"varint,1,opt,name=eventId,proto3" json:"eventId,omitempty"` - Timestamp *timestamppb.Timestamp `protobuf:"bytes,2,opt,name=timestamp,proto3" json:"timestamp,omitempty"` -} - -func (x *TestEvent) Reset() { - *x = TestEvent{} - if protoimpl.UnsafeEnabled { - mi := &file_test_proto_msgTypes[0] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *TestEvent) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*TestEvent) ProtoMessage() {} - -func (x *TestEvent) ProtoReflect() protoreflect.Message { - mi := &file_test_proto_msgTypes[0] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use TestEvent.ProtoReflect.Descriptor instead. -func (*TestEvent) Descriptor() ([]byte, []int) { - return file_test_proto_rawDescGZIP(), []int{0} -} - -func (x *TestEvent) GetEventId() int32 { - if x != nil { - return x.EventId - } - return 0 -} - -func (x *TestEvent) GetTimestamp() *timestamppb.Timestamp { - if x != nil { - return x.Timestamp - } - return nil -} - -var File_test_proto protoreflect.FileDescriptor - -var file_test_proto_rawDesc = []byte{ - 0x0a, 0x0a, 0x74, 0x65, 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1f, 0x67, 0x6f, - 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x74, 0x69, - 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x5f, 0x0a, - 0x09, 0x54, 0x65, 0x73, 0x74, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x65, 0x76, - 0x65, 0x6e, 0x74, 0x49, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x07, 0x65, 0x76, 0x65, - 0x6e, 0x74, 0x49, 0x64, 0x12, 0x38, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, - 0x70, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, - 0x61, 0x6d, 0x70, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x42, 0x41, - 0x5a, 0x3f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x64, 0x61, 0x70, - 0x72, 0x2f, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x73, 0x2d, 0x63, 0x6f, 0x6e, - 0x74, 0x72, 0x69, 0x62, 0x2f, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2f, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x2f, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2f, 0x73, 0x71, 0x6c, 0x73, 0x65, 0x72, 0x76, 0x65, - 0x72, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, -} - -var ( - file_test_proto_rawDescOnce sync.Once - file_test_proto_rawDescData = file_test_proto_rawDesc -) - -func file_test_proto_rawDescGZIP() []byte { - file_test_proto_rawDescOnce.Do(func() { - file_test_proto_rawDescData = protoimpl.X.CompressGZIP(file_test_proto_rawDescData) - }) - return file_test_proto_rawDescData -} - -var file_test_proto_msgTypes = make([]protoimpl.MessageInfo, 1) -var file_test_proto_goTypes = []interface{}{ - (*TestEvent)(nil), // 0: TestEvent - (*timestamppb.Timestamp)(nil), // 1: google.protobuf.Timestamp -} -var file_test_proto_depIdxs = []int32{ - 1, // 0: TestEvent.timestamp:type_name -> google.protobuf.Timestamp - 1, // [1:1] is the sub-list for method output_type - 1, // [1:1] is the sub-list for method input_type - 1, // [1:1] is the sub-list for extension type_name - 1, // [1:1] is the sub-list for extension extendee - 0, // [0:1] is the sub-list for field type_name -} - -func init() { file_test_proto_init() } -func file_test_proto_init() { - if File_test_proto != nil { - return - } - if !protoimpl.UnsafeEnabled { - file_test_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TestEvent); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - } - type x struct{} - out := protoimpl.TypeBuilder{ - File: protoimpl.DescBuilder{ - GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: file_test_proto_rawDesc, - NumEnums: 0, - NumMessages: 1, - NumExtensions: 0, - NumServices: 0, - }, - GoTypes: file_test_proto_goTypes, - DependencyIndexes: file_test_proto_depIdxs, - MessageInfos: file_test_proto_msgTypes, - }.Build() - File_test_proto = out.File - file_test_proto_rawDesc = nil - file_test_proto_goTypes = nil - file_test_proto_depIdxs = nil -} diff --git a/common/proto/state/sqlserver/test.proto b/common/proto/state/sqlserver/test.proto deleted file mode 100644 index 7c84802598..0000000000 --- a/common/proto/state/sqlserver/test.proto +++ /dev/null @@ -1,10 +0,0 @@ -syntax = "proto3"; - -option go_package = "github.com/dapr/components-contrib/common/proto/state/sqlserver"; - -import "google/protobuf/timestamp.proto"; - -message TestEvent { - int32 eventId = 1; - google.protobuf.Timestamp timestamp = 2; -} \ No newline at end of file diff --git a/state/sqlserver/sqlserver.go b/state/sqlserver/sqlserver.go index f607d47af1..35e2f7ed20 100644 --- a/state/sqlserver/sqlserver.go +++ b/state/sqlserver/sqlserver.go @@ -16,7 +16,6 @@ package sqlserver import ( "context" "database/sql" - "encoding/base64" "encoding/hex" "encoding/json" "errors" @@ -288,15 +287,8 @@ func (s *SQLServer) Get(ctx context.Context, req *state.GetRequest) (*state.GetR } } - bytes, err := base64.StdEncoding.DecodeString(data) - if err != nil { - s.logger. - WithFields(map[string]any{"error": err}). - Debug("error decoding base64 data. Fallback to []byte") - bytes = []byte(data) - } return &state.GetResponse{ - Data: bytes, + Data: []byte(data), ETag: ptr.Of(etag), Metadata: metadata, }, nil @@ -313,23 +305,16 @@ type dbExecutor interface { } func (s *SQLServer) executeSet(ctx context.Context, db dbExecutor, req *state.SetRequest) error { - var reqValue string - - bytes, ok := req.Value.([]byte) - if !ok { - bt, err := json.Marshal(req.Value) - if err != nil { - return err - } - reqValue = string(bt) - } else { - reqValue = base64.StdEncoding.EncodeToString(bytes) + var err error + var bytes []byte + bytes, err = utils.Marshal(req.Value, json.Marshal) + if err != nil { + return err } - etag := sql.Named(rowVersionColumnName, nil) if req.HasETag() { var b []byte - b, err := hex.DecodeString(*req.ETag) + b, err = hex.DecodeString(*req.ETag) if err != nil { return state.NewETagError(state.ETagInvalid, err) } @@ -342,14 +327,13 @@ func (s *SQLServer) executeSet(ctx context.Context, db dbExecutor, req *state.Se } var res sql.Result - var err error if req.Options.Concurrency == state.FirstWrite { res, err = db.ExecContext(ctx, s.upsertCommand, sql.Named(keyColumnName, req.Key), - sql.Named("Data", reqValue), etag, + sql.Named("Data", string(bytes)), etag, sql.Named("FirstWrite", 1), sql.Named("TTL", ttl)) } else { res, err = db.ExecContext(ctx, s.upsertCommand, sql.Named(keyColumnName, req.Key), - sql.Named("Data", reqValue), etag, + sql.Named("Data", string(bytes)), etag, sql.Named("FirstWrite", 0), sql.Named("TTL", ttl)) } diff --git a/state/sqlserver/sqlserver_integration_test.go b/state/sqlserver/sqlserver_integration_test.go index e317128a40..7af6220476 100644 --- a/state/sqlserver/sqlserver_integration_test.go +++ b/state/sqlserver/sqlserver_integration_test.go @@ -30,12 +30,10 @@ import ( "testing" "time" - "github.com/google/uuid" + uuid "github.com/google/uuid" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "google.golang.org/protobuf/proto" - "github.com/dapr/components-contrib/common/proto/state/sqlserver" "github.com/dapr/components-contrib/metadata" "github.com/dapr/components-contrib/state" "github.com/dapr/kit/logger" @@ -44,7 +42,7 @@ import ( const ( // connectionStringEnvKey defines the key containing the integration test connection string // To use docker, server=localhost;user id=sa;password=Pass@Word1;port=1433; - // To use Azure SQL, server=.database.windows.net;User id=;port=1433;password=;database=dapr_test;. + // To use Azure SQL, server=.database.windows.net;user id=;port=1433;password=;database=dapr_test;. connectionStringEnvKey = "DAPR_TEST_SQL_CONNSTRING" usersTableName = "Users" beverageTea = "tea" @@ -79,7 +77,6 @@ func TestIntegrationCases(t *testing.T) { t.Run("Multi operations", testMultiOperations) t.Run("Insert and Update Set Record Dates", testInsertAndUpdateSetRecordDates) t.Run("Multiple initializations", testMultipleInitializations) - t.Run("Should preserve byte data when not base64 encoded", testNonBase64ByteData) // Run concurrent set tests 10 times const executions = 10 @@ -115,9 +112,6 @@ func createMetadata(schema string, kt KeyType, indexedProperties string) state.M // Ensure the database is running // For docker, use: docker run --name sqlserver -e "ACCEPT_EULA=Y" -e "SA_PASSWORD=Pass@Word1" -p 1433:1433 -d mcr.microsoft.com/mssql/server:2019-GA-ubuntu-16.04. -// For azure-sql-edge use: -// docker volume create sqlvolume -// docker run --name sqlserver -e "ACCEPT_EULA=Y" -e "MSSQL_SA_PASSWORD=Pass@Word1" -e "MSSQL_PID=Developer" -e "MSSQL_AGENT_ENABLED=TRUE" -e "MSSQL_COLLATION=SQL_Latin1_General_CP1_CI_AS" -e "MSSQL_LCID=1033" -p 1433:1433 -v sqlvolume:/var/opt/mssql -d mcr.microsoft.com/azure-sql-edge:latest func getTestStore(t *testing.T, indexedProperties string) *SQLServer { return getTestStoreWithKeyType(t, StringKeyType, indexedProperties) } @@ -603,23 +597,3 @@ func testMultipleInitializations(t *testing.T) { }) } } - -func testNonBase64ByteData(t *testing.T) { - t.Run("Set And Get", func(t *testing.T) { - store := getTestStore(t, "") - request := &sqlserver.TestEvent{ - EventId: -1, - } - requestBytes, err := proto.Marshal(request) - require.NoError(t, err) - require.NoError(t, store.Set(t.Context(), &state.SetRequest{Key: "1", Value: requestBytes})) - resp, err := store.Get(t.Context(), &state.GetRequest{Key: "1"}) - require.NoError(t, err) - - response := &sqlserver.TestEvent{} - err = proto.Unmarshal(resp.Data, response) - require.NoError(t, err) - - assert.EqualValues(t, request.GetEventId(), response.GetEventId()) - }) -} From b714d656a727b412bddd01de32bcc6c6c5ff9f05 Mon Sep 17 00:00:00 2001 From: swatimodi-scout <159426020+swatimodi-scout@users.noreply.github.com> Date: Thu, 14 Aug 2025 10:41:29 +0530 Subject: [PATCH 22/30] Added enpoint in configuration (#3931) Signed-off-by: swatimodi-scout Co-authored-by: peterdalinis-scout Co-authored-by: Yaron Schneider Signed-off-by: swatimodi-scout --- bindings/aws/kinesis/kinesis.go | 1 + 1 file changed, 1 insertion(+) diff --git a/bindings/aws/kinesis/kinesis.go b/bindings/aws/kinesis/kinesis.go index 32c7b01829..9881818278 100644 --- a/bindings/aws/kinesis/kinesis.go +++ b/bindings/aws/kinesis/kinesis.go @@ -156,6 +156,7 @@ func (a *AWSKinesis) Read(ctx context.Context, handler bindings.Handler) (err er if a.closed.Load() { return errors.New("binding is closed") } + if a.metadata.KinesisConsumerMode == SharedThroughput { // Configure the KCL worker with custom endpoints for LocalStack config := a.authProvider.Kinesis().WorkerCfg(ctx, a.streamName, a.consumerName, a.consumerMode) From 3bf76a6aa2039e58dea34995feb058823426736c Mon Sep 17 00:00:00 2001 From: Nelson Parente Date: Thu, 14 Aug 2025 16:39:10 +0100 Subject: [PATCH 23/30] fix: pin macos-14 (#3972) Signed-off-by: nelson.parente Signed-off-by: swatimodi-scout --- .github/workflows/components-contrib-all.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/components-contrib-all.yml b/.github/workflows/components-contrib-all.yml index 25875b7ded..332a14bc01 100644 --- a/.github/workflows/components-contrib-all.yml +++ b/.github/workflows/components-contrib-all.yml @@ -68,19 +68,20 @@ jobs: GOLANGCI_LINT_VER: "v1.64.6" strategy: matrix: - os: [ubuntu-latest, windows-latest, macOS-latest] + # TODO: @nelson-parente Upgrade macos latest once dns is fixed. + os: [ubuntu-latest, windows-latest, macos-14] target_arch: [arm, amd64] include: - os: ubuntu-latest target_os: linux - os: windows-latest target_os: windows - - os: macOS-latest + - os: macos-14 target_os: darwin exclude: - os: windows-latest target_arch: arm - - os: macOS-latest + - os: macos-14 target_arch: arm steps: - name: Set default payload repo and ref From 04a677e5d3b5226f33c93cb674366ae49d8a6b13 Mon Sep 17 00:00:00 2001 From: Oliver Tomlinson Date: Thu, 14 Aug 2025 23:03:50 +0100 Subject: [PATCH 24/30] fixed the kubernetes secret component for cryptography (#3971) Signed-off-by: Oliver Tomlinson Co-authored-by: Yaron Schneider Signed-off-by: swatimodi-scout --- crypto/kubernetes/secrets/component.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crypto/kubernetes/secrets/component.go b/crypto/kubernetes/secrets/component.go index f6d2573470..da442ecf29 100644 --- a/crypto/kubernetes/secrets/component.go +++ b/crypto/kubernetes/secrets/component.go @@ -123,7 +123,7 @@ func (k *kubeSecretsCrypto) retrieveKeyFromSecret(parentCtx context.Context, key // parseKeyString returns the secret name, key, and optional namespace from the key parameter. // If the key parameter doesn't contain a namespace, returns the default one. func (k *kubeSecretsCrypto) parseKeyString(param string) (namespace string, secret string, key string, err error) { - parts := strings.Split(key, "/") + parts := strings.Split(param, "/") switch len(parts) { case 3: namespace = parts[0] From dff826a4b8deafb4aa8c1ce1287340188580268c Mon Sep 17 00:00:00 2001 From: Mike Nguyen Date: Mon, 18 Aug 2025 21:51:29 +0100 Subject: [PATCH 25/30] fix: pubsub.azure.eventhubs status check (#3976) Signed-off-by: Mike Nguyen Signed-off-by: swatimodi-scout --- .../pubsub/azure/eventhubs/send-iot-device-events.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/certification/pubsub/azure/eventhubs/send-iot-device-events.sh b/tests/certification/pubsub/azure/eventhubs/send-iot-device-events.sh index 13662b2fcd..02bf80528e 100755 --- a/tests/certification/pubsub/azure/eventhubs/send-iot-device-events.sh +++ b/tests/certification/pubsub/azure/eventhubs/send-iot-device-events.sh @@ -34,9 +34,10 @@ az login --service-principal -u $AzureCertificationServicePrincipalClientId -p $ # Create test device ID if not already present IOT_HUB_TEST_DEVICE_NAME="certification-test-device" -if [[ -z "$(az iot hub device-identity show -n ${AzureIotHubName} -d ${IOT_HUB_TEST_DEVICE_NAME})" ]]; then +az iot hub device-identity show -n ${AzureIotHubName} -d ${IOT_HUB_TEST_DEVICE_NAME} >/dev/null 2>&1 +if [[ $? -ne 0 ]]; then az iot hub device-identity create -n ${AzureIotHubName} -d ${IOT_HUB_TEST_DEVICE_NAME} - sleep 5 + sleep 10 fi # Send the test IoT device messages to the IoT Hub.`testmessageForEventHubCertificationTest` is being asserted in the certification test From b3272112798895b2807467e7b0e572b81a6d8154 Mon Sep 17 00:00:00 2001 From: Mike Nguyen Date: Mon, 18 Aug 2025 22:02:47 +0100 Subject: [PATCH 26/30] fix(tests): fix state.azure.tablestorage cert test (#3975) Signed-off-by: Mike Nguyen Co-authored-by: Yaron Schneider Signed-off-by: swatimodi-scout --- .../azure/tablestorage/components/aadtest/azuretablekey.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/certification/state/azure/tablestorage/components/aadtest/azuretablekey.yaml b/tests/certification/state/azure/tablestorage/components/aadtest/azuretablekey.yaml index 5006e7ce43..46b4fa8bbc 100644 --- a/tests/certification/state/azure/tablestorage/components/aadtest/azuretablekey.yaml +++ b/tests/certification/state/azure/tablestorage/components/aadtest/azuretablekey.yaml @@ -15,7 +15,5 @@ spec: key: AzureBlobStorageAccessKey - name: tableName value: "certificationTable" - - name: serviceURL - value: "https://dapr3ctstorage.table.core.windows.net" auth: secretStore: envvar-secret-store \ No newline at end of file From e240fba8ec54a423e79703449deaa902ee7672ca Mon Sep 17 00:00:00 2001 From: Sam Date: Mon, 18 Aug 2025 16:04:07 -0500 Subject: [PATCH 27/30] fix(test): up ngrok version + fix silent failure on url parsing (#3969) Signed-off-by: Samantha Coyle Co-authored-by: Yaron Schneider Signed-off-by: swatimodi-scout --- ...formance-bindings.azure.eventgrid-setup.sh | 42 +++++- bindings/azure/eventgrid/eventgrid.go | 120 +++++++++++++++--- 2 files changed, 140 insertions(+), 22 deletions(-) diff --git a/.github/scripts/components-scripts/conformance-bindings.azure.eventgrid-setup.sh b/.github/scripts/components-scripts/conformance-bindings.azure.eventgrid-setup.sh index 9dcea682d0..9898257c33 100755 --- a/.github/scripts/components-scripts/conformance-bindings.azure.eventgrid-setup.sh +++ b/.github/scripts/components-scripts/conformance-bindings.azure.eventgrid-setup.sh @@ -2,16 +2,48 @@ set -e -# Start ngrok -wget https://bin.equinox.io/c/4VmDzA7iaHb/ngrok-stable-linux-amd64.zip -unzip -qq ngrok-stable-linux-amd64.zip +echo "Downloading ngrok..." +wget https://bin.equinox.io/c/bNyj1mQVY4c/ngrok-v3-stable-linux-amd64.tgz +tar -xzf ngrok-v3-stable-linux-amd64.tgz ./ngrok authtoken ${AzureEventGridNgrokToken} -./ngrok http -log=stdout --log-level debug -host-header=localhost 9000 > /tmp/ngrok.log & + +echo "Starting ngrok tunnel..." +./ngrok http --log=stdout --log-level=debug --host-header=localhost 9000 > /tmp/ngrok.log 2>&1 & +NGROK_PID=$! + +echo "Waiting for ngrok to start..." sleep 10 -NGROK_ENDPOINT=`cat /tmp/ngrok.log | grep -Eom1 'https://.*' | sed 's/\s.*//'` +# Ensure ngrok is still running +if ! kill -0 $NGROK_PID 2>/dev/null; then + echo "Ngrok process died. Log output:" + cat /tmp/ngrok.log + exit 1 +fi + +echo "Getting tunnel URL from ngrok API..." +MAX_RETRIES=30 +RETRY_COUNT=0 + +while [ $RETRY_COUNT -lt $MAX_RETRIES ]; do + if curl -s http://localhost:4040/api/tunnels > /tmp/ngrok_api.json 2>/dev/null; then + # Extract the public URL from the API response + NGROK_ENDPOINT=$(cat /tmp/ngrok_api.json | grep -o '"public_url":"[^"]*"' | head -1 | cut -d'"' -f4) + + if [ -n "$NGROK_ENDPOINT" ] && [ "$NGROK_ENDPOINT" != "null" ]; then + echo "Successfully got tunnel URL from API: $NGROK_ENDPOINT" + break + fi + fi + + echo "Waiting for ngrok API to be ready... (attempt $((RETRY_COUNT + 1))/$MAX_RETRIES)" + sleep 2 + RETRY_COUNT=$((RETRY_COUNT + 1)) +done + echo "Ngrok endpoint: ${NGROK_ENDPOINT}" echo "AzureEventGridSubscriberEndpoint=${NGROK_ENDPOINT}/api/events" >> $GITHUB_ENV +echo "Ngrok log:" cat /tmp/ngrok.log # Schedule trigger to kill ngrok diff --git a/bindings/azure/eventgrid/eventgrid.go b/bindings/azure/eventgrid/eventgrid.go index cbe47c1442..f6d26b321c 100644 --- a/bindings/azure/eventgrid/eventgrid.go +++ b/bindings/azure/eventgrid/eventgrid.go @@ -51,9 +51,14 @@ const ( // Format for the "jwks_uri" endpoint // The %s refers to the tenant ID jwksURIFormat = "https://login.microsoftonline.com/%s/discovery/v2.0/keys" - // Format for the "iss" claim in the JWT + // Format for the "iss" claim in the JWT (Azure AD v2.0) // The %s refers to the tenant ID jwtIssuerFormat = "https://login.microsoftonline.com/%s/v2.0" + // Format for the "iss" claim in the JWT (Azure AD v1.0 - legacy) + // The %s refers to the tenant ID + jwtIssuerV1Format = "https://sts.windows.net/%s/" + // Eventgrid managed identity issuer + eventGridIssuer = "https://eventgrid.azure.net" ) // AzureEventGrid allows sending/receiving Azure Event Grid events. @@ -111,12 +116,14 @@ func (a *AzureEventGrid) Init(_ context.Context, metadata bindings.Metadata) err var matchAuthHeader = regexp.MustCompile(`(?i)^(Bearer )?(([A-Za-z0-9_-]+\.){2}[A-Za-z0-9_-]+)$`) -func (a *AzureEventGrid) validateAuthHeader(ctx context.Context, authorizationHeader string) bool { +func (a *AzureEventGrid) validateAuthHeader(ctx *fasthttp.RequestCtx) bool { // Extract the bearer token from the header + authorizationHeader := string(ctx.Request.Header.Peek("authorization")) if authorizationHeader == "" { a.logger.Error("Incoming webhook request does not contain an Authorization header") return false } + match := matchAuthHeader.FindStringSubmatch(authorizationHeader) if len(match) < 3 { a.logger.Error("Incoming webhook request does not contain a valid bearer token in the Authorization header") @@ -124,21 +131,98 @@ func (a *AzureEventGrid) validateAuthHeader(ctx context.Context, authorizationHe } token := match[2] - // Validate the JWT - _, err := jwt.ParseString( - token, - jwt.WithKeySet(a.jwks, jws.WithInferAlgorithmFromKey(true)), - jwt.WithAudience(a.metadata.azureClientID), - jwt.WithIssuer(fmt.Sprintf(jwtIssuerFormat, a.metadata.azureTenantID)), - jwt.WithAcceptableSkew(5*time.Minute), - jwt.WithContext(ctx), - ) + // First, parse the JWT to see what claims we received + parsedToken, err := jwt.ParseString(token, jwt.WithVerify(false)) if err != nil { - a.logger.Errorf("Failed to validate JWT in the incoming webhook request: %v", err) + a.logger.Errorf("Failed to parse JWT: %v", err) return false } - return true + actualIssuer := parsedToken.Issuer() + azureADV2Issuer := fmt.Sprintf(jwtIssuerFormat, a.metadata.azureTenantID) + expectedAudience := a.metadata.azureClientID + switch actualIssuer { + case azureADV2Issuer: + // AzureAD v2.0 issuer + _, err = jwt.ParseString( + token, + jwt.WithKeySet(a.jwks, jws.WithInferAlgorithmFromKey(true)), + jwt.WithAudience(expectedAudience), + jwt.WithIssuer(azureADV2Issuer), + jwt.WithAcceptableSkew(5*time.Minute), + jwt.WithContext(context.Background()), + ) + if err == nil { + return true + } + + // Also check webhook URL as audience + _, err = jwt.ParseString( + token, + jwt.WithKeySet(a.jwks, jws.WithInferAlgorithmFromKey(true)), + jwt.WithAudience(a.metadata.SubscriberEndpoint), + jwt.WithIssuer(azureADV2Issuer), + jwt.WithAcceptableSkew(5*time.Minute), + jwt.WithContext(context.Background()), + ) + if err == nil { + return true + } + + a.logger.Errorf("JWT validation failed for AzureAD v2.0 issuer") + return false + + case fmt.Sprintf(jwtIssuerV1Format, a.metadata.azureTenantID): + // AzureAD v1.0 issuer + a.logger.Infof("Detected AzureAD v1.0 issuer, validating...") + _, err = jwt.ParseString( + token, + jwt.WithKeySet(a.jwks, jws.WithInferAlgorithmFromKey(true)), + jwt.WithAudience(expectedAudience), + jwt.WithIssuer(actualIssuer), + jwt.WithAcceptableSkew(5*time.Minute), + jwt.WithContext(context.Background()), + ) + if err == nil { + return true + } + _, err = jwt.ParseString( + token, + jwt.WithKeySet(a.jwks, jws.WithInferAlgorithmFromKey(true)), + jwt.WithAudience(a.metadata.SubscriberEndpoint), + jwt.WithIssuer(actualIssuer), + jwt.WithAcceptableSkew(5*time.Minute), + jwt.WithContext(context.Background()), + ) + if err == nil { + return true + } + + a.logger.Errorf("JWT validation failed for AzureAD v1.0 issuer") + return false + + case eventGridIssuer: + // eventgrid managed identity issuer - use webhook URL as audience + _, err = jwt.ParseString( + token, + jwt.WithKeySet(a.jwks, jws.WithInferAlgorithmFromKey(true)), + jwt.WithAudience(a.metadata.SubscriberEndpoint), + jwt.WithIssuer(eventGridIssuer), + jwt.WithAcceptableSkew(5*time.Minute), + jwt.WithContext(context.Background()), + ) + if err == nil { + return true + } + + a.logger.Errorf("JWT validation failed for eventgrid issuer: %v", err) + return false + + default: + a.logger.Errorf("Unexpected JWT issuer: %s. Expected either '%s', '%s', or '%s'", + actualIssuer, azureADV2Issuer, fmt.Sprintf(jwtIssuerV1Format, a.metadata.azureTenantID), eventGridIssuer) + return false + } } // Initializes the JWKS cache @@ -284,10 +368,12 @@ func (a *AzureEventGrid) requestHandler(handler bindings.Handler) fasthttp.Reque return } - // Validate the Authorization header - authorizationHeader := string(ctx.Request.Header.Peek("authorization")) - // Note that ctx is a fasthttp context so it's actually tied to the server's lifecycle and not the request's - if !a.validateAuthHeader(ctx, authorizationHeader) { + // Options requests (webhook validation handshake) don't require authentication + // Azure Event Grid sends options requests without authorization header during initial validation + if method == http.MethodOptions { + // Skip authentication for options requests + } else if !a.validateAuthHeader(ctx) { + // Note that ctx is a fasthttp context so it's actually tied to the server's lifecycle and not the request's ctx.Response.Header.SetStatusCode(http.StatusUnauthorized) _, err = ctx.Response.BodyWriter().Write([]byte("401 Unauthorized")) if err != nil { From eaf31b05ab1e45f13cbe90b996667ed68d39a192 Mon Sep 17 00:00:00 2001 From: Eileen Yu <48944635+Eileen-Yu@users.noreply.github.com> Date: Mon, 18 Aug 2025 14:05:20 -0700 Subject: [PATCH 28/30] fix: postgresql doc link fix (#3904) Signed-off-by: Eileen Yu Co-authored-by: Yaron Schneider Signed-off-by: swatimodi-scout --- state/postgresql/v1/metadata.yaml | 4 ++-- state/postgresql/v2/metadata.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/state/postgresql/v1/metadata.yaml b/state/postgresql/v1/metadata.yaml index 597bd0259a..a06f93a9ab 100644 --- a/state/postgresql/v1/metadata.yaml +++ b/state/postgresql/v1/metadata.yaml @@ -7,7 +7,7 @@ status: stable title: "PostgreSQL" urls: - title: Reference - url: https://docs.dapr.io/reference/components-reference/supported-state-stores/setup-postgresql/ + url: https://docs.dapr.io/reference/components-reference/supported-state-stores/setup-postgresql-v1/ capabilities: # If actorStateStore is present, the metadata key actorStateStore can be used - actorStateStore @@ -175,4 +175,4 @@ metadata: required: false description: The path to the SSL root certificate file example: "/path/to/ssl/root/cert.pem" - type: string \ No newline at end of file + type: string diff --git a/state/postgresql/v2/metadata.yaml b/state/postgresql/v2/metadata.yaml index cf894d195c..b1b1d59e57 100644 --- a/state/postgresql/v2/metadata.yaml +++ b/state/postgresql/v2/metadata.yaml @@ -7,7 +7,7 @@ status: stable title: "PostgreSQL" urls: - title: Reference - url: https://docs.dapr.io/reference/components-reference/supported-state-stores/setup-postgresql/ + url: https://docs.dapr.io/reference/components-reference/supported-state-stores/setup-postgresql-v2/ capabilities: # If actorStateStore is present, the metadata key actorStateStore can be used - actorStateStore From 5f9743e972541883050f2a86111b5ed8056f7362 Mon Sep 17 00:00:00 2001 From: Sam Date: Mon, 18 Aug 2025 16:06:40 -0500 Subject: [PATCH 29/30] feat: add state component registrations (#3951) Signed-off-by: Samantha Coyle Co-authored-by: Yaron Schneider Signed-off-by: swatimodi-scout --- conversation/echo/metadata.yaml | 11 ++ state/aerospike/aerospike.go | 6 +- state/aerospike/metadata.yaml | 26 ++++ state/alicloud/tablestore/metadata.yaml | 40 +++++ state/couchbase/couchbase.go | 12 +- state/couchbase/metadata.yaml | 43 ++++++ state/etcd/etcd.go | 2 +- state/etcd/metadata.yaml | 61 ++++++++ state/gcp/firestore/firestore.go | 5 +- state/gcp/firestore/metadata.yaml | 98 ++++++++++++ state/hashicorp/consul/metadata.yaml | 36 +++++ state/hazelcast/hazelcast.go | 4 +- state/hazelcast/metadata.yaml | 21 +++ state/jetstream/metadata.yaml | 37 +++++ state/oci/objectstorage/metadata.yaml | 84 ++++++++++ state/oci/objectstorage/objectstorage.go | 3 +- state/oracledatabase/metadata.yaml | 30 ++++ state/oracledatabase/oracledatabaseaccess.go | 6 +- state/rethinkdb/metadata.yaml | 152 +++++++++++++++++++ state/rethinkdb/rethinkdb.go | 101 +++++++++++- state/sqlite/metadata.yaml | 53 +++++++ state/zookeeper/metadata.yaml | 39 +++++ state/zookeeper/zk.go | 1 + 23 files changed, 845 insertions(+), 26 deletions(-) create mode 100644 conversation/echo/metadata.yaml create mode 100644 state/aerospike/metadata.yaml create mode 100644 state/alicloud/tablestore/metadata.yaml create mode 100644 state/couchbase/metadata.yaml create mode 100644 state/etcd/metadata.yaml create mode 100644 state/gcp/firestore/metadata.yaml create mode 100644 state/hashicorp/consul/metadata.yaml create mode 100644 state/hazelcast/metadata.yaml create mode 100644 state/jetstream/metadata.yaml create mode 100644 state/oci/objectstorage/metadata.yaml create mode 100644 state/oracledatabase/metadata.yaml create mode 100644 state/rethinkdb/metadata.yaml create mode 100644 state/sqlite/metadata.yaml create mode 100644 state/zookeeper/metadata.yaml diff --git a/conversation/echo/metadata.yaml b/conversation/echo/metadata.yaml new file mode 100644 index 0000000000..d5cf348862 --- /dev/null +++ b/conversation/echo/metadata.yaml @@ -0,0 +1,11 @@ +# yaml-language-server: $schema=../../../component-metadata-schema.json +schemaVersion: v1 +type: conversation +name: echo +version: v1 +status: alpha +title: "Echo" +urls: + - title: Reference + url: https://docs.dapr.io/reference/components-reference/supported-conversation/echo/ +metadata: [] diff --git a/state/aerospike/aerospike.go b/state/aerospike/aerospike.go index 27206519b6..ea95c1516d 100644 --- a/state/aerospike/aerospike.go +++ b/state/aerospike/aerospike.go @@ -34,9 +34,9 @@ import ( ) type aerospikeMetadata struct { - Hosts string - Namespace string - Set string // optional + Hosts string `json:"hosts"` + Namespace string `json:"namespace"` + Set string `json:"set"` // optional } var ( diff --git a/state/aerospike/metadata.yaml b/state/aerospike/metadata.yaml new file mode 100644 index 0000000000..379cbf74f1 --- /dev/null +++ b/state/aerospike/metadata.yaml @@ -0,0 +1,26 @@ +# yaml-language-server: $schema=../../../component-metadata-schema.json +schemaVersion: v1 +type: state +name: aerospike +version: v1 +status: alpha +title: "Aerospike" +urls: + - title: Reference + url: https://docs.dapr.io/reference/components-reference/supported-state-stores/setup-aerospike/ +metadata: + - name: hosts + type: string + required: true + description: Comma-separated list of Aerospike hosts. + example: "localhost:3000,aerospike:3000" + - name: namespace + type: string + required: true + description: The Aerospike namespace. + example: "test" + - name: set + type: string + required: false + description: The Aerospike set name. + example: "dapr-state" \ No newline at end of file diff --git a/state/alicloud/tablestore/metadata.yaml b/state/alicloud/tablestore/metadata.yaml new file mode 100644 index 0000000000..42469c5dea --- /dev/null +++ b/state/alicloud/tablestore/metadata.yaml @@ -0,0 +1,40 @@ +# yaml-language-server: $schema=../../../component-metadata-schema.json +schemaVersion: v1 +type: state +name: alicloud.tablestore +version: v1 +status: alpha +title: "AliCloud TableStore" +urls: + - title: Reference + url: https://docs.dapr.io/reference/components-reference/supported-state-stores/ +authenticationProfiles: + - title: "Access Key Authentication" + description: "Authenticate using an access key" + metadata: + - name: accessKeyID + type: string + required: true + description: The access key ID of the AliCloud TableStore instance. + example: "my_access_key_id" + - name: accessKey + type: string + required: true + description: The access key of the AliCloud TableStore instance. + example: "my_access_key" +metadata: + - name: endpoint + type: string + required: true + description: The endpoint of the AliCloud TableStore instance. + example: "https://tablestore.aliyuncs.com" + - name: instanceName + type: string + required: true + description: The name of the AliCloud TableStore instance. + example: "my_instance" + - name: tableName + type: string + required: true + description: The name of the table to use. + example: "my_table" diff --git a/state/couchbase/couchbase.go b/state/couchbase/couchbase.go index 32264c78ef..562f748286 100644 --- a/state/couchbase/couchbase.go +++ b/state/couchbase/couchbase.go @@ -57,12 +57,12 @@ type Couchbase struct { } type couchbaseMetadata struct { - CouchbaseURL string - Username string - Password string - BucketName string - NumReplicasDurableReplication uint - NumReplicasDurablePersistence uint + CouchbaseURL string `json:"couchbaseURL"` + Username string `json:"username"` + Password string `json:"password"` + BucketName string `json:"bucketName"` + NumReplicasDurableReplication uint `json:"numReplicasDurableReplication"` + NumReplicasDurablePersistence uint `json:"numReplicasDurablePersistence"` } // NewCouchbaseStateStore returns a new couchbase state store. diff --git a/state/couchbase/metadata.yaml b/state/couchbase/metadata.yaml new file mode 100644 index 0000000000..6b30d80448 --- /dev/null +++ b/state/couchbase/metadata.yaml @@ -0,0 +1,43 @@ +# yaml-language-server: $schema=../../../component-metadata-schema.json +schemaVersion: v1 +type: state +name: couchbase +version: v1 +status: alpha +title: "Couchbase" +urls: + - title: Reference + url: https://docs.dapr.io/reference/components-reference/supported-state-stores/setup-couchbase/ +authenticationProfiles: + - title: "Connection String" + description: "Connect to Couchbase using a connection string." + metadata: + - name: couchbaseURL + type: string + required: true + description: The Couchbase connection string. + example: "couchbase://localhost" + - name: username + type: string + required: true + - name: password + type: string + required: true + - name: bucketName + type: string + required: true + description: The Couchbase bucket name. + example: "default" +metadata: + - name: numReplicasDurableReplication + type: integer + required: false + description: The number of replicas for durable replication. + example: 1 + default: 0 + - name: numReplicasDurablePersistence + type: integer + required: false + description: The number of replicas for durable persistence. + example: 1 + default: 0 \ No newline at end of file diff --git a/state/etcd/etcd.go b/state/etcd/etcd.go index 67272071ee..b7a1db3a69 100644 --- a/state/etcd/etcd.go +++ b/state/etcd/etcd.go @@ -123,7 +123,7 @@ func (e *Etcd) ParseClientFromConfig(etcdConfig *etcdConfig) (*clientv3.Client, config := clientv3.Config{ Endpoints: endpoints, - DialTimeout: 5 * time.Second, + DialTimeout: 5 * time.Second, // TODO: make this configurable TLS: tlsConfig, } client, err := clientv3.New(config) diff --git a/state/etcd/metadata.yaml b/state/etcd/metadata.yaml new file mode 100644 index 0000000000..671a91442f --- /dev/null +++ b/state/etcd/metadata.yaml @@ -0,0 +1,61 @@ +# yaml-language-server: $schema=../../../component-metadata-schema.json +schemaVersion: v1 +type: state +name: etcd +version: v1 +status: alpha +title: "etcd" +urls: + - title: Reference + url: https://docs.dapr.io/reference/components-reference/supported-state-stores/setup-etcd/ +authenticationProfiles: + - title: "TLS Authentication" + description: "Connect to etcd using TLS encryption with certificates." + metadata: + - name: tlsEnable + type: string + required: true + description: Enable TLS authentication. + example: "true" + - name: ca + type: string + required: true + description: CA certificate for TLS verification. + example: | + -----BEGIN CERTIFICATE----- + XXX + -----END CERTIFICATE----- + - name: cert + type: string + required: true + description: Client certificate for TLS authentication. + example: | + -----BEGIN CERTIFICATE----- + XXX + -----END CERTIFICATE----- + - name: key + type: string + required: true + description: Client private key for TLS authentication. + example: | + -----BEGIN PRIVATE KEY----- + XXX + -----END PRIVATE KEY----- +metadata: + - name: endpoints + type: string + required: true + description: Comma-separated list of etcd endpoints. + example: "localhost:2379" + - name: keyPrefixPath + type: string + required: false + description: The key prefix path to use. + example: "my_key_prefix_path" + default: "" + - name: maxTxnOps + type: integer + required: false + description: Maximum number of operations allowed in a transaction. + example: 128 + default: 128 diff --git a/state/gcp/firestore/firestore.go b/state/gcp/firestore/firestore.go index 2e4b8eb62e..6157c6646f 100644 --- a/state/gcp/firestore/firestore.go +++ b/state/gcp/firestore/firestore.go @@ -47,6 +47,9 @@ type Firestore struct { } type firestoreMetadata struct { + // TODO: update these to use camel case instead in future + + // TODO: rm type field? It's stable component but this field is NOT used anywhere except in tests. Type string `json:"type"` ProjectID string `json:"project_id" mapstructure:"project_id"` PrivateKeyID string `json:"private_key_id" mapstructure:"private_key_id"` @@ -219,7 +222,7 @@ func (f *Firestore) Close() error { return nil } -func getGCPClient(ctx context.Context, metadata *firestoreMetadata, l logger.Logger) (*datastore.Client, error) { +func getGCPClient(_ context.Context, metadata *firestoreMetadata, l logger.Logger) (*datastore.Client, error) { var gcpClient *datastore.Client var err error diff --git a/state/gcp/firestore/metadata.yaml b/state/gcp/firestore/metadata.yaml new file mode 100644 index 0000000000..b489334518 --- /dev/null +++ b/state/gcp/firestore/metadata.yaml @@ -0,0 +1,98 @@ +# yaml-language-server: $schema=../../../component-metadata-schema.json +schemaVersion: v1 +type: state +name: gcp.firestore +version: v1 +status: stable +title: "GCP Firestore" +urls: + - title: Reference + url: https://docs.dapr.io/reference/components-reference/supported-state-stores/setup-firestore/ +authenticationProfiles: + - title: "Service Account Authentication" + description: "Authenticate using a GCP service account JSON key." + metadata: + - name: project_id + type: string + required: true + description: The GCP project ID. + example: "my-project" + - name: private_key_id + type: string + required: true + description: The private key ID from the service account JSON. + example: "abc123def456" + - name: private_key + type: string + required: true + description: The private key from the service account JSON. + example: | + -----BEGIN PRIVATE KEY----- + XXX + -----END PRIVATE KEY----- + - name: client_email + type: string + required: true + description: The client email from the service account JSON. + example: "firestore@my-project.iam.gserviceaccount.com" + - name: client_id + type: string + required: true + description: The client ID from the service account JSON. + example: "123456789012345678901" + - name: auth_uri + type: string + required: true + description: The authentication URI from the service account JSON. + example: "https://accounts.google.com/o/oauth2/auth" + - name: token_uri + type: string + required: true + description: The token URI from the service account JSON. + example: "https://oauth2.googleapis.com/token" + - name: auth_provider_x509_cert_url + type: string + required: true + description: The auth provider certificate URL from the service account JSON. + example: "https://www.googleapis.com/oauth2/v1/certs" + - name: client_x509_cert_url + type: string + required: true + description: The client certificate URL from the service account JSON. + example: "https://www.googleapis.com/robot/v1/metadata/x509/firestore%40my-project.iam.gserviceaccount.com" + - title: "Implicit Credentials" + description: "Authenticate using implicit credentials (Application Default Credentials) for local development." + metadata: + - name: project_id + type: string + required: true + description: The GCP project ID. + example: "my-project" + - name: endpoint + type: string + required: false + description: The Firestore endpoint URL (for custom endpoints or emulator). + example: "https://firestore.googleapis.com" +metadata: + - name: entity_kind + type: string + required: false + description: The entity kind (collection name) for storing state data. + example: "dapr-state" + default: "DaprState" + - name: type + type: string + required: false + description: The type of service account. + example: "service_account" + - name: connectionEndpoint + type: string + required: false + description: The Firestore connection endpoint. + example: "https://firestore.googleapis.com" + - name: noIndex + type: bool + required: false + description: Whether to disable indexing for the entity. + example: false + default: false diff --git a/state/hashicorp/consul/metadata.yaml b/state/hashicorp/consul/metadata.yaml new file mode 100644 index 0000000000..f02e4975f5 --- /dev/null +++ b/state/hashicorp/consul/metadata.yaml @@ -0,0 +1,36 @@ +# yaml-language-server: $schema=../../../component-metadata-schema.json +schemaVersion: v1 +type: state +name: hashicorp.consul +version: v1 +status: alpha +title: "Echo" +urls: + - title: Reference + url: https://docs.dapr.io/reference/components-reference/supported-state-stores/setup-consul/ +metadata: + - name: datacenter + type: string + required: true + description: The datacenter to use. + example: "dc1" + - name: httpAddr + type: string + required: true + description: The HTTP address of the Consul server. + example: "http://localhost:8500" + - name: aclToken + type: string + required: true + description: The ACL token to use. + example: "my_acl_token" + - name: scheme + type: string + required: true + description: The scheme to use. + example: "http" + - name: keyPrefixPath + type: string + required: true + description: The key prefix path to use. + example: "my_key_prefix_path" diff --git a/state/hazelcast/hazelcast.go b/state/hazelcast/hazelcast.go index 2fe56f0488..9ef11e08a9 100644 --- a/state/hazelcast/hazelcast.go +++ b/state/hazelcast/hazelcast.go @@ -40,8 +40,8 @@ type Hazelcast struct { } type hazelcastMetadata struct { - HazelcastServers string - HazelcastMap string + HazelcastServers string `json:"hazelcastServers" mapstructure:"hazelcastServers"` + HazelcastMap string `json:"hazelcastMap" mapstructure:"hazelcastMap"` } // NewHazelcastStore returns a new hazelcast backed state store. diff --git a/state/hazelcast/metadata.yaml b/state/hazelcast/metadata.yaml new file mode 100644 index 0000000000..f0e3b495f2 --- /dev/null +++ b/state/hazelcast/metadata.yaml @@ -0,0 +1,21 @@ +# yaml-language-server: $schema=../../../component-metadata-schema.json +schemaVersion: v1 +type: state +name: hazelcast +version: v1 +status: alpha +title: "Hazelcast" +urls: + - title: Reference + url: https://docs.dapr.io/reference/components-reference/supported-state-stores/setup-hazelcast/ +metadata: + - name: hazelcastServers + type: string + required: true + description: Comma-separated list of Hazelcast servers. + example: "localhost:5701" + - name: hazelcastMap + type: string + required: true + description: The Hazelcast map name. + example: "dapr-state" diff --git a/state/jetstream/metadata.yaml b/state/jetstream/metadata.yaml new file mode 100644 index 0000000000..24311881bd --- /dev/null +++ b/state/jetstream/metadata.yaml @@ -0,0 +1,37 @@ +# yaml-language-server: $schema=../../../component-metadata-schema.json +schemaVersion: v1 +type: state +name: jetstream +version: v1 +status: alpha +title: "JetStream KV" +urls: + - title: Reference + url: https://docs.dapr.io/reference/components-reference/supported-state-stores/setup-jetstream-kv/ +metadata: + - name: name + type: string + required: false + description: The name of the bucket to use. + example: "my_bucket" + default: "dapr.io - statestore.jetstream" + - name: natsURL + type: string + required: true + description: The NATS URL to use. + example: "nats://localhost:4222" + - name: jwt + type: string + required: false + description: The JWT to use. + example: "my_jwt" + - name: seedKey + type: string + required: false + description: The seed key to use. + example: "my_seed_key" + - name: bucket + type: string + required: true + description: The bucket to use. + example: "my_bucket" diff --git a/state/oci/objectstorage/metadata.yaml b/state/oci/objectstorage/metadata.yaml new file mode 100644 index 0000000000..896831a050 --- /dev/null +++ b/state/oci/objectstorage/metadata.yaml @@ -0,0 +1,84 @@ +# yaml-language-server: $schema=../../../component-metadata-schema.json +schemaVersion: v1 +type: state +name: oci.objectstorage +version: v1 +status: alpha +title: "OCI Object Storage" +urls: + - title: Reference + url: https://docs.dapr.io/reference/components-reference/supported-state-stores/setup-oci-objectstorage/ +authenticationProfiles: + - title: "API Key Authentication" + description: "Authenticate using OCI API key with user OCID, fingerprint, and private key." + metadata: + - name: region + type: string + required: true + description: The OCI region where the bucket is located. + example: "us-ashburn-1" + - name: userOCID + type: string + required: true + description: The OCID of the user. + example: "ocid1.user.oc1..example" + - name: fingerPrint + type: string + required: true + description: The fingerprint of the API key. + example: "xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx" + - name: privateKey + type: string + required: true + description: The private key for API authentication. + example: "-----BEGIN PRIVATE KEY-----\n...\n-----END PRIVATE KEY-----" + - name: tenancyOCID + type: string + required: true + description: The OCID of the tenancy. + example: "ocid1.tenancy.oc1..example" + - title: "Instance Principal Authentication" + description: "Authenticate using OCI instance principal (automatically available on OCI compute instances)." + metadata: + - name: instancePrincipalAuthentication + type: bool + required: true + description: Set to true to use instance principal authentication. + example: true + default: false + - title: "Configuration File Authentication" + description: "Authenticate using OCI configuration file." + metadata: + - name: configFileAuthentication + type: bool + required: true + description: Set to true to use configuration file authentication. + example: true + default: falsee + - name: configFilePath + type: string + required: true + description: Path to the OCI configuration file. + example: "~/.oci/config" + - name: configFileProfile + type: string + required: false + description: Profile name in the OCI configuration file. + example: "DEFAULT" + default: "" +metadata: + - name: bucketName + type: string + required: true + description: The name of the OCI Object Storage bucket. + example: "my-bucket" + - name: compartmentOCID + type: string + required: true + description: The OCID of the compartment containing the bucket. + example: "ocid1.compartment.oc1..example" + - name: namespace + type: string + required: false + description: The OCI namespace for the object storage. + example: "my-namespace" diff --git a/state/oci/objectstorage/objectstorage.go b/state/oci/objectstorage/objectstorage.go index d2d87accb1..147ef472c9 100644 --- a/state/oci/objectstorage/objectstorage.go +++ b/state/oci/objectstorage/objectstorage.go @@ -51,7 +51,6 @@ const ( privateKeyKey = "privateKey" userKey = "userOCID" bucketNameKey = "bucketName" - metadataTTLKey = "ttlInSeconds" daprStateStoreMetaLabel = "dapr-state-store" expiryTimeMetaLabel = "expiry-time-from-ttl" isoDateTimeFormat = "2006-01-02T15:04:05" @@ -80,7 +79,7 @@ type objectStoreMetadata struct { InstancePrincipalAuthentication bool ConfigFileAuthentication bool - OCIObjectStorageClient *objectstorage.ObjectStorageClient + OCIObjectStorageClient *objectstorage.ObjectStorageClient `mapstructure:"-"` } type objectStoreClient interface { diff --git a/state/oracledatabase/metadata.yaml b/state/oracledatabase/metadata.yaml new file mode 100644 index 0000000000..3912170f99 --- /dev/null +++ b/state/oracledatabase/metadata.yaml @@ -0,0 +1,30 @@ +# yaml-language-server: $schema=../../../component-metadata-schema.json +schemaVersion: v1 +type: state +name: oracledatabase +version: v1 +status: alpha +title: "Oracle Database" +urls: + - title: Reference + url: https://docs.dapr.io/reference/components-reference/supported-state-stores/setup-oracledatabase/ +authenticationProfiles: + - title: "Connection String" + description: "Connect to Oracle Database using a connection string." + metadata: + - name: connectionString + type: string + required: true +metadata: + - name: tableName + type: string + required: false + description: The table name to store state data. + example: "dapr_state" + default: "state" + - name: oracleWalletLocation + type: string + required: false + description: The location of the Oracle wallet. + example: "/path/to/wallet" + default: "" diff --git a/state/oracledatabase/oracledatabaseaccess.go b/state/oracledatabase/oracledatabaseaccess.go index fdd5374a59..42a9c77aa6 100644 --- a/state/oracledatabase/oracledatabaseaccess.go +++ b/state/oracledatabase/oracledatabaseaccess.go @@ -50,9 +50,9 @@ type oracleDatabaseAccess struct { } type oracleDatabaseMetadata struct { - ConnectionString string - OracleWalletLocation string - TableName string + ConnectionString string `json:"connectionString"` + OracleWalletLocation string `json:"oracleWalletLocation"` + TableName string `json:"tableName"` } // newOracleDatabaseAccess creates a new instance of oracleDatabaseAccess. diff --git a/state/rethinkdb/metadata.yaml b/state/rethinkdb/metadata.yaml new file mode 100644 index 0000000000..e12181cf5b --- /dev/null +++ b/state/rethinkdb/metadata.yaml @@ -0,0 +1,152 @@ +# yaml-language-server: $schema=../../../component-metadata-schema.json +schemaVersion: v1 +type: state +name: rethinkdb +version: v1 +status: beta +title: "RethinkDB" +urls: + - title: Reference + url: https://docs.dapr.io/reference/components-reference/supported-state-stores/setup-rethinkdb/ +authenticationProfiles: + - title: "Username/Password Authentication" + description: "Authenticate using username and password." + metadata: + - name: address + type: string + required: false + description: The RethinkDB server address. + example: "localhost:28015" + - name: addresses + type: string + required: false + description: Comma-separated list of RethinkDB server addresses. + example: "localhost:28015,localhost:28016" + - name: database + type: string + required: true + description: The RethinkDB database name. + example: "dapr" + default: "" + - name: username + type: string + required: false + description: The username for authentication. If not provided, the admin user is used for v1 handshake protocol. + example: "admin" + - name: password + type: string + required: false + description: The password for authentication. This is only used for v1 handshake protocol. + example: "password" + - title: "TLS Authentication" + description: "Authenticate using client certificate and key." + metadata: + - name: enableTLS + type: bool + required: false + description: Whether to enable TLS encryption. + example: false + default: false + - name: clientCert + type: string + required: true + description: The client certificate for TLS authentication. + example: "-----BEGIN CERTIFICATE-----\nXXX..." + - name: clientKey + type: string + required: true + description: The client key for TLS authentication. + example: "-----BEGIN PRIVATE KEY-----\nXXX..." + sensitive: true +metadata: + - name: table + type: string + required: false + description: The table name to store state data. + example: "daprstate" + default: "daprstate" + - name: archive + type: bool + required: false + description: Whether to archive changes to a separate table. + example: false + default: false + - name: timeout + type: string + required: false + description: Connection timeout duration. + example: "10s" + - name: useJSONNumber + type: bool + required: false + description: Whether to use json.Number instead of float64. + example: false + default: false + - name: numRetries + type: number + required: false + description: Number of times to retry queries on connection errors. + example: 3 + - name: hostDecayDuration + type: string + required: false + description: Host decay duration for weighted host selection. + example: "5m" + default: "5m" + - name: useOpentracing + type: bool + required: false + description: Whether to enable opentracing for queries. + example: false + default: false + - name: writeTimeout + type: string + required: false + description: Write timeout duration." + example: "10s" + - name: readTimeout + type: string + required: false + description: Read timeout duration." + example: "10s" + - name: handshakeVersion + type: number + required: false + description: Handshake version for RethinkDB." + example: 1 + - name: keepAlivePeriod + type: string + required: false + description: Keep alive period duration." + example: "30s" + - name: maxIdle + type: number + required: false + description: Maximum number of idle connections." + example: 5 + - name: authKey + type: string + required: false + description: The authentication key for RethinkDB." + example: "auth-key" + sensitive: true + - name: initialCap + type: number + required: false + description: Initial connection pool capacity." + example: 5 + - name: maxOpen + type: number + required: false + description: Maximum number of open connections." + example: 10 + - name: discoverHosts + type: bool + required: false + description: Whether to discover hosts." + example: false + - name: nodeRefreshInterval + type: string + required: false + description: Node refresh interval duration." + example: "5m" diff --git a/state/rethinkdb/rethinkdb.go b/state/rethinkdb/rethinkdb.go index 83f93adf6f..0c6e5b5297 100644 --- a/state/rethinkdb/rethinkdb.go +++ b/state/rethinkdb/rethinkdb.go @@ -15,6 +15,7 @@ package rethinkdb import ( "context" + "crypto/tls" "encoding/json" "errors" "fmt" @@ -32,9 +33,12 @@ import ( ) const ( - stateTableNameDefault = "daprstate" - stateTablePKName = "id" - stateArchiveTableName = "daprstate_archive" + stateTableNameDefault = "daprstate" + // TODO: this needs to be exposed as a metadata option? + stateTablePKName = "id" + // TODO: this needs to be exposed as a metadata option + stateArchiveTableName = "daprstate_archive" + // TODO: this needs to be exposed as a metadata option? stateArchiveTablePKName = "key" ) @@ -47,9 +51,39 @@ type RethinkDB struct { } type stateConfig struct { - r.ConnectOpts `mapstructure:",squash"` - Archive bool `json:"archive"` - Table string `json:"table"` + ConnectOptsWrapper `mapstructure:",squash"` + Archive bool `json:"archive"` + Table string `json:"table"` +} + +// ConnectOptsWrapper wraps r.ConnectOpts but excludes TLSConfig +// This is needed because the metadata decoder does not support nested structs with tags as inputs in the metadata.yaml file +type ConnectOptsWrapper struct { + Address string `gorethink:"address,omitempty"` + Addresses []string `gorethink:"addresses,omitempty"` + Database string `gorethink:"database,omitempty"` + Username string `gorethink:"username,omitempty"` + Password string `gorethink:"password,omitempty"` + AuthKey string `gorethink:"authkey,omitempty"` + Timeout time.Duration `gorethink:"timeout,omitempty"` + WriteTimeout time.Duration `gorethink:"write_timeout,omitempty"` + ReadTimeout time.Duration `gorethink:"read_timeout,omitempty"` + KeepAlivePeriod time.Duration `gorethink:"keep_alive_timeout,omitempty"` + HandshakeVersion int `gorethink:"handshake_version,omitempty"` + MaxIdle int `gorethink:"max_idle,omitempty"` + InitialCap int `gorethink:"initial_cap,omitempty"` + MaxOpen int `gorethink:"max_open,omitempty"` + DiscoverHosts bool `gorethink:"discover_hosts,omitempty"` + NodeRefreshInterval time.Duration `gorethink:"node_refresh_interval,omitempty"` + UseJSONNumber bool `gorethink:"use_json_number,omitempty"` + NumRetries int `gorethink:"num_retries,omitempty"` + HostDecayDuration time.Duration `gorethink:"host_decay_duration,omitempty"` + UseOpentracing bool `gorethink:"use_opentracing,omitempty"` + + // TLS fields must be brought in as separate fields as they will not be processed by the metadata decoder properly without this + EnableTLS bool `gorethink:"enable_tls,omitempty"` + ClientCert string `gorethink:"client_cert,omitempty"` + ClientKey string `gorethink:"client_key,omitempty"` } type stateRecord struct { @@ -81,7 +115,41 @@ func (s *RethinkDB) Init(ctx context.Context, metadata state.Metadata) error { if s.session != nil && s.session.IsConnected() { s.session.Close() } - ses, err := r.Connect(cfg.ConnectOpts) + + // Convert wrapper to r.ConnectOpts + connectOpts := r.ConnectOpts{ + Address: cfg.Address, + Addresses: cfg.Addresses, + Database: cfg.Database, + Username: cfg.Username, + Password: cfg.Password, + AuthKey: cfg.AuthKey, + Timeout: cfg.Timeout, + WriteTimeout: cfg.WriteTimeout, + ReadTimeout: cfg.ReadTimeout, + KeepAlivePeriod: cfg.KeepAlivePeriod, + HandshakeVersion: r.HandshakeVersion(cfg.HandshakeVersion), + MaxIdle: cfg.MaxIdle, + InitialCap: cfg.InitialCap, + MaxOpen: cfg.MaxOpen, + DiscoverHosts: cfg.DiscoverHosts, + NodeRefreshInterval: cfg.NodeRefreshInterval, + UseJSONNumber: cfg.UseJSONNumber, + NumRetries: cfg.NumRetries, + HostDecayDuration: cfg.HostDecayDuration, + UseOpentracing: cfg.UseOpentracing, + } + + // Configure TLS if enabled + if cfg.EnableTLS { + tlsConfig, tlsErr := createTLSConfig(cfg.ClientCert, cfg.ClientKey) + if tlsErr != nil { + return fmt.Errorf("error creating TLS config: %w", tlsErr) + } + connectOpts.TLSConfig = tlsConfig + } + + ses, err := r.Connect(connectOpts) if err != nil { return fmt.Errorf("error connecting to the database: %w", err) } @@ -293,7 +361,7 @@ func (s *RethinkDB) BulkDelete(ctx context.Context, req []state.DeleteRequest, _ return nil } -func metadataToConfig(cfg map[string]string, logger logger.Logger) (*stateConfig, error) { +func metadataToConfig(cfg map[string]string, _ logger.Logger) (*stateConfig, error) { // defaults c := stateConfig{ Table: stateTableNameDefault, @@ -307,6 +375,23 @@ func metadataToConfig(cfg map[string]string, logger logger.Logger) (*stateConfig return &c, nil } +// createTLSConfig creates a tls.Config from client certificate and key +func createTLSConfig(clientCert, clientKey string) (*tls.Config, error) { + if clientCert == "" || clientKey == "" { + return nil, errors.New("both client certificate and key are required for TLS") + } + + cert, err := tls.X509KeyPair([]byte(clientCert), []byte(clientKey)) + if err != nil { + return nil, fmt.Errorf("error parsing client certificate and key: %w", err) + } + + return &tls.Config{ + Certificates: []tls.Certificate{cert}, + MinVersion: tls.VersionTLS12, + }, nil +} + func (s *RethinkDB) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { metadataStruct := stateConfig{} metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.StateStoreType) diff --git a/state/sqlite/metadata.yaml b/state/sqlite/metadata.yaml new file mode 100644 index 0000000000..8c1e6aead0 --- /dev/null +++ b/state/sqlite/metadata.yaml @@ -0,0 +1,53 @@ +# yaml-language-server: $schema=../../../component-metadata-schema.json +schemaVersion: v1 +type: state +name: sqlite +version: v1 +status: stable +title: "SQLite" +urls: + - title: Reference + url: https://docs.dapr.io/reference/components-reference/supported-state-stores/setup-sqlite/ +authenticationProfiles: + - title: "Connection String" + description: "Authenticate using a connection string." + metadata: + - name: connectionString + type: string + required: true + description: The SQLite database connection string. +metadata: + - name: timeout + type: string + required: false + description: Timeout for database requests in seconds. + example: "20s" + default: "20s" + - name: busyTimeout + type: string + required: false + description: Busy timeout for database operations in seconds. + example: "2s" + default: "2s" + - name: disableWAL + type: bool + required: false + description: Disable WAL journaling. Should not use WAL if database is stored on a network filesystem. + example: false + default: false + - name: tableName + type: string + required: false + description: The name of the table to store state data. + example: "state" + - name: metadataTableName + type: string + required: false + description: The name of the table to store metadata. + example: "metadata" + - name: cleanupInterval + type: string + required: false + description: Interval for cleanup operations in seconds. Set to 0 to disable. + example: "0s" + default: "0s" diff --git a/state/zookeeper/metadata.yaml b/state/zookeeper/metadata.yaml new file mode 100644 index 0000000000..d9b381da86 --- /dev/null +++ b/state/zookeeper/metadata.yaml @@ -0,0 +1,39 @@ +# yaml-language-server: $schema=../../../component-metadata-schema.json +schemaVersion: v1 +type: state +name: zookeeper +version: v1 +status: alpha +title: "ZooKeeper" +urls: + - title: Reference + url: https://docs.dapr.io/reference/components-reference/supported-state-stores/setup-zookeeper/ +metadata: + - name: servers + type: string + required: true + description: Comma-separated list of ZooKeeper servers. + example: "localhost:2181" + - name: sessionTimeout + type: string + required: true + description: Session timeout in seconds. + example: "10s" + - name: maxBufferSize + type: integer + required: false + description: The maximum buffer size in bytes. + example: 1048576 + default: 1048576 # 1MB + - name: maxConnBufferSize + type: integer + required: false + description: The maximum connection buffer size in bytes. + example: 1048576 + default: 1048576 # 1MB + - name: keyPrefixPath + type: string + required: false + description: The key prefix path to use. + example: "my_key_prefix_path" + default: "" diff --git a/state/zookeeper/zk.go b/state/zookeeper/zk.go index 418a44b8d3..52bd54793a 100644 --- a/state/zookeeper/zk.go +++ b/state/zookeeper/zk.go @@ -32,6 +32,7 @@ import ( "github.com/dapr/kit/ptr" ) +// TODO: I think we need more defaults set on these metadata fields. const ( anyVersion = -1 defaultMaxBufferSize = 1024 * 1024 From 07178fb5e26966473b2f268987bcf8b341910329 Mon Sep 17 00:00:00 2001 From: swatimodi-scout Date: Wed, 20 Aug 2025 07:21:51 +0000 Subject: [PATCH 30/30] Formatted gofumpt File Signed-off-by: swatimodi-scout --- common/authentication/aws/client.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/common/authentication/aws/client.go b/common/authentication/aws/client.go index 06d47bb3a0..df97e5f9c7 100644 --- a/common/authentication/aws/client.go +++ b/common/authentication/aws/client.go @@ -184,8 +184,7 @@ func (c *KinesisClients) Stream(ctx context.Context, streamName string) (*string /** * If the error is not nil, do not proceed to the next step * as it may cause a nil pointer error on stream.StreamDescription.StreamARN. - */ - if err != nil { + */ if err != nil { return nil, err } return stream.StreamDescription.StreamARN, err