chore: Read destination metadata path from config (#127)

alexluong · web-flow · commit de6cc33f61dc · 2024-12-03T14:42:04.000+07:00
* chore: Read destination metadata path from config

* chore: Custom webhook instructions for docker compose example
diff --git a/examples/docker-compose/compose.yml b/examples/docker-compose/compose.yml
@@ -10,6 +10,8 @@ services:
         condition: service_healthy
     ports:
       - "${PORT}:${PORT}"
+    volumes:
+      - ./config/outpost:/config/outpost
     environment:
       SERVICE: api
 
diff --git a/examples/docker-compose/config/outpost/destinations/webhook/instructions.md b/examples/docker-compose/config/outpost/destinations/webhook/instructions.md
@@ -0,0 +1,3 @@
+# Webhook
+
+My custom webhook instructions!
diff --git a/internal/config/config.go b/internal/config/config.go
@@ -44,6 +44,7 @@ type Config struct {
 	RetryMaxCount                   int
 	LogBatcherDelayThresholdSeconds int
 	LogBatcherItemCountThreshold    int
+	DestinationMetadataPath         string
 }
 
 var defaultConfig = map[string]any{
@@ -63,6 +64,7 @@ var defaultConfig = map[string]any{
 	"MAX_RETRY_COUNT":                     10,
 	"LOG_BATCHER_DELAY_THRESHOLD_SECONDS": 5,
 	"LOG_BATCHER_ITEM_COUNT_THRESHOLD":    100,
+	"DESTINATION_METADATA_PATH":           "config/outpost/destinations",
 }
 
 var (
@@ -179,6 +181,7 @@ func Parse(flags Flags) (*Config, error) {
 		RetryMaxCount:                   mustInt(viper, "MAX_RETRY_COUNT"),
 		LogBatcherDelayThresholdSeconds: mustInt(viper, "LOG_BATCHER_DELAY_THRESHOLD_SECONDS"),
 		LogBatcherItemCountThreshold:    mustInt(viper, "LOG_BATCHER_ITEM_COUNT_THRESHOLD"),
+		DestinationMetadataPath:         viper.GetString("DESTINATION_METADATA_PATH"),
 	}
 
 	return config, nil
diff --git a/internal/destregistry/baseprovider.go b/internal/destregistry/baseprovider.go
@@ -17,8 +17,7 @@ type BaseProvider struct {
 }
 
 // NewBaseProvider creates a new base provider with loaded metadata
-func NewBaseProvider(providerType string) (*BaseProvider, error) {
-	loader := metadata.NewMetadataLoader("")
+func NewBaseProvider(loader *metadata.MetadataLoader, providerType string) (*BaseProvider, error) {
 	meta, err := loader.Load(providerType)
 	if err != nil {
 		return nil, fmt.Errorf("loading provider metadata: %w", err)
diff --git a/internal/destregistry/providers/default.go b/internal/destregistry/providers/default.go
@@ -8,19 +8,21 @@ import (
 )
 
 func RegisterDefault(registry destregistry.Registry) error {
-	aws, err := destaws.New()
+	loader := registry.MetadataLoader()
+
+	aws, err := destaws.New(loader)
 	if err != nil {
 		return err
 	}
 	registry.RegisterProvider("aws", aws)
 
-	rabbitmq, err := destrabbitmq.New()
+	rabbitmq, err := destrabbitmq.New(loader)
 	if err != nil {
 		return err
 	}
 	registry.RegisterProvider("rabbitmq", rabbitmq)
 
-	webhook, err := destwebhook.New()
+	webhook, err := destwebhook.New(loader)
 	if err != nil {
 		return err
 	}
diff --git a/internal/destregistry/providers/destaws/destaws.go b/internal/destregistry/providers/destaws/destaws.go
@@ -11,6 +11,7 @@ import (
 	"github.com/aws/aws-sdk-go-v2/service/sqs/types"
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/hookdeck/outpost/internal/destregistry"
+	"github.com/hookdeck/outpost/internal/destregistry/metadata"
 	"github.com/hookdeck/outpost/internal/models"
 )
 
@@ -32,8 +33,8 @@ type AWSDestinationCredentials struct {
 
 var _ destregistry.Provider = (*AWSDestination)(nil)
 
-func New() (*AWSDestination, error) {
-	base, err := destregistry.NewBaseProvider("aws")
+func New(loader *metadata.MetadataLoader) (*AWSDestination, error) {
+	base, err := destregistry.NewBaseProvider(loader, "aws")
 	if err != nil {
 		return nil, err
 	}
diff --git a/internal/destregistry/providers/destaws/destaws_test.go b/internal/destregistry/providers/destaws/destaws_test.go
@@ -36,7 +36,7 @@ func TestAWSDestination_Validate(t *testing.T) {
 		}),
 	)
 
-	awsDestination, err := destaws.New()
+	awsDestination, err := destaws.New(testutil.Registry.MetadataLoader())
 	require.NoError(t, err)
 
 	t.Run("should validate valid destination", func(t *testing.T) {
@@ -118,7 +118,7 @@ func TestIntegrationAWSDestination_Publish(t *testing.T) {
 	require.NoError(t, err)
 	queueURL, err := awsutil.EnsureQueue(context.Background(), sqsClient, mq.AWSSQS.Topic, nil)
 	require.NoError(t, err)
-	awsdestination, err := destaws.New()
+	awsdestination, err := destaws.New(testutil.Registry.MetadataLoader())
 	require.NoError(t, err)
 
 	destination := testutil.DestinationFactory.Any(
diff --git a/internal/destregistry/providers/destrabbitmq/destrabbitmq.go b/internal/destregistry/providers/destrabbitmq/destrabbitmq.go
@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 
 	"github.com/hookdeck/outpost/internal/destregistry"
+	"github.com/hookdeck/outpost/internal/destregistry/metadata"
 	"github.com/hookdeck/outpost/internal/models"
 	"github.com/rabbitmq/amqp091-go"
 )
@@ -25,8 +26,8 @@ type RabbitMQDestinationCredentials struct {
 
 var _ destregistry.Provider = (*RabbitMQDestination)(nil)
 
-func New() (*RabbitMQDestination, error) {
-	base, err := destregistry.NewBaseProvider("rabbitmq")
+func New(loader *metadata.MetadataLoader) (*RabbitMQDestination, error) {
+	base, err := destregistry.NewBaseProvider(loader, "rabbitmq")
 	if err != nil {
 		return nil, err
 	}
diff --git a/internal/destregistry/providers/destrabbitmq/destrabbitmq_test.go b/internal/destregistry/providers/destrabbitmq/destrabbitmq_test.go
@@ -33,7 +33,7 @@ func TestRabbitMQDestination_Validate(t *testing.T) {
 		}),
 	)
 
-	rabbitmqDestination, err := destrabbitmq.New()
+	rabbitmqDestination, err := destrabbitmq.New(testutil.Registry.MetadataLoader())
 	require.NoError(t, err)
 
 	t.Run("should validate valid destination", func(t *testing.T) {
@@ -108,7 +108,7 @@ func TestRabbitMQDestination_Validate(t *testing.T) {
 func TestRabbitMQDestination_Publish(t *testing.T) {
 	t.Parallel()
 
-	rabbitmqDestination, err := destrabbitmq.New()
+	rabbitmqDestination, err := destrabbitmq.New(testutil.Registry.MetadataLoader())
 	require.NoError(t, err)
 
 	destination := testutil.DestinationFactory.Any(
@@ -139,7 +139,7 @@ func TestIntegrationRabbitMQDestination_Publish(t *testing.T) {
 	t.Cleanup(testinfra.Start(t))
 
 	mq := testinfra.NewMQRabbitMQConfig(t)
-	rabbitmqDestination, err := destrabbitmq.New()
+	rabbitmqDestination, err := destrabbitmq.New(testutil.Registry.MetadataLoader())
 	require.NoError(t, err)
 
 	destination := testutil.DestinationFactory.Any(
diff --git a/internal/destregistry/providers/destwebhook/destwebhook.go b/internal/destregistry/providers/destwebhook/destwebhook.go
@@ -10,6 +10,7 @@ import (
 	"net/http"
 
 	"github.com/hookdeck/outpost/internal/destregistry"
+	"github.com/hookdeck/outpost/internal/destregistry/metadata"
 	"github.com/hookdeck/outpost/internal/models"
 )
 
@@ -23,8 +24,8 @@ type WebhookDestinationConfig struct {
 
 var _ destregistry.Provider = (*WebhookDestination)(nil)
 
-func New() (*WebhookDestination, error) {
-	base, err := destregistry.NewBaseProvider("webhook")
+func New(loader *metadata.MetadataLoader) (*WebhookDestination, error) {
+	base, err := destregistry.NewBaseProvider(loader, "webhook")
 	if err != nil {
 		return nil, err
 	}
diff --git a/internal/destregistry/providers/destwebhook/destwebhook_test.go b/internal/destregistry/providers/destwebhook/destwebhook_test.go
@@ -27,7 +27,7 @@ func TestWebhookDestination_Validate(t *testing.T) {
 		testutil.DestinationFactory.WithCredentials(map[string]string{}),
 	)
 
-	webhookDestination, err := destwebhook.New()
+	webhookDestination, err := destwebhook.New(testutil.Registry.MetadataLoader())
 	require.NoError(t, err)
 
 	t.Run("should validate valid destination", func(t *testing.T) {
@@ -139,7 +139,7 @@ func (suite *webhookDestinationSuite) TeardownTest(t *testing.T) {
 func TestWebhookDestination_Publish(t *testing.T) {
 	t.Parallel()
 
-	webhookDestination, err := destwebhook.New()
+	webhookDestination, err := destwebhook.New(testutil.Registry.MetadataLoader())
 	require.NoError(t, err)
 
 	destination := testutil.DestinationFactory.Any(
diff --git a/internal/destregistry/registry.go b/internal/destregistry/registry.go
@@ -13,6 +13,7 @@ type Registry interface {
 	GetProvider(destinationType string) (Provider, error)
 	RetrieveProviderMetadata(providerType string) (*metadata.ProviderMetadata, error)
 	ListProviderMetadata() map[string]*metadata.ProviderMetadata
+	MetadataLoader() *metadata.MetadataLoader
 }
 
 type Provider interface {
@@ -27,14 +28,20 @@ type Provider interface {
 }
 
 type registry struct {
-	providers map[string]Provider
-	metadata  map[string]*metadata.ProviderMetadata
+	providers      map[string]Provider
+	metadata       map[string]*metadata.ProviderMetadata
+	metadataLoader *metadata.MetadataLoader
 }
 
-func NewRegistry() Registry {
+type Config struct {
+	DestinationMetadataPath string
+}
+
+func NewRegistry(cfg *Config) Registry {
 	return &registry{
-		providers: make(map[string]Provider),
-		metadata:  make(map[string]*metadata.ProviderMetadata),
+		providers:      make(map[string]Provider),
+		metadata:       make(map[string]*metadata.ProviderMetadata),
+		metadataLoader: metadata.NewMetadataLoader(cfg.DestinationMetadataPath),
 	}
 }
 
@@ -68,3 +75,7 @@ func (r *registry) ListProviderMetadata() map[string]*metadata.ProviderMetadata
 	}
 	return metadataCopy
 }
+
+func (r *registry) MetadataLoader() *metadata.MetadataLoader {
+	return r.metadataLoader
+}
diff --git a/internal/services/api/api.go b/internal/services/api/api.go
@@ -42,7 +42,9 @@ type APIService struct {
 func NewService(ctx context.Context, wg *sync.WaitGroup, cfg *config.Config, logger *otelzap.Logger) (*APIService, error) {
 	wg.Add(1)
 
-	registry := destregistry.NewRegistry()
+	registry := destregistry.NewRegistry(&destregistry.Config{
+		DestinationMetadataPath: cfg.DestinationMetadataPath,
+	})
 	if err := destregistrydefault.RegisterDefault(registry); err != nil {
 		return nil, err
 	}
diff --git a/internal/services/delivery/delivery.go b/internal/services/delivery/delivery.go
@@ -62,7 +62,9 @@ func NewService(ctx context.Context,
 	cleanupFuncs = append(cleanupFuncs, cleanupLogMQ)
 
 	if handler == nil {
-		registry := destregistry.NewRegistry()
+		registry := destregistry.NewRegistry(&destregistry.Config{
+			DestinationMetadataPath: cfg.DestinationMetadataPath,
+		})
 		if err := destregistrydefault.RegisterDefault(registry); err != nil {
 			return nil, err
 		}
diff --git a/internal/util/testutil/registry.go b/internal/util/testutil/registry.go
@@ -8,6 +8,8 @@ import (
 var Registry destregistry.Registry
 
 func init() {
-	Registry = destregistry.NewRegistry()
+	Registry = destregistry.NewRegistry(&destregistry.Config{
+		DestinationMetadataPath: "",
+	})
 	destregistrydefault.RegisterDefault(Registry)
 }

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+# Webhook`
	`2`	`+`
	`3`	`+My custom webhook instructions!`
Original file line number	Diff line number	Diff line change
`@@ -44,6 +44,7 @@ type Config struct {`
`44`	`44`	`RetryMaxCount int`
`45`	`45`	`LogBatcherDelayThresholdSeconds int`
`46`	`46`	`LogBatcherItemCountThreshold int`
	`47`	`+ DestinationMetadataPath string`
`47`	`48`	`}`
`48`	`49`
`49`	`50`	`var defaultConfig = map[string]any{`
`@@ -63,6 +64,7 @@ var defaultConfig = map[string]any{`
`63`	`64`	`"MAX_RETRY_COUNT": 10,`
`64`	`65`	`"LOG_BATCHER_DELAY_THRESHOLD_SECONDS": 5,`
`65`	`66`	`"LOG_BATCHER_ITEM_COUNT_THRESHOLD": 100,`
	`67`	`+ "DESTINATION_METADATA_PATH": "config/outpost/destinations",`
`66`	`68`	`}`
`67`	`69`
`68`	`70`	`var (`
`@@ -179,6 +181,7 @@ func Parse(flags Flags) (*Config, error) {`
`179`	`181`	`RetryMaxCount: mustInt(viper, "MAX_RETRY_COUNT"),`
`180`	`182`	`LogBatcherDelayThresholdSeconds: mustInt(viper, "LOG_BATCHER_DELAY_THRESHOLD_SECONDS"),`
`181`	`183`	`LogBatcherItemCountThreshold: mustInt(viper, "LOG_BATCHER_ITEM_COUNT_THRESHOLD"),`
	`184`	`+ DestinationMetadataPath: viper.GetString("DESTINATION_METADATA_PATH"),`
`182`	`185`	`}`
`183`	`186`
`184`	`187`	`return config, nil`
Original file line number	Diff line number	Diff line change
`@@ -17,8 +17,7 @@ type BaseProvider struct {`
`17`	`17`	`}`
`18`	`18`
`19`	`19`	`// NewBaseProvider creates a new base provider with loaded metadata`
`20`		`-func NewBaseProvider(providerType string) (*BaseProvider, error) {`
`21`		`- loader := metadata.NewMetadataLoader("")`
	`20`	`+func NewBaseProvider(loader metadata.MetadataLoader, providerType string) (BaseProvider, error) {`
`22`	`21`	`meta, err := loader.Load(providerType)`
`23`	`22`	`if err != nil {`
`24`	`23`	`return nil, fmt.Errorf("loading provider metadata: %w", err)`
Original file line number	Diff line number	Diff line change
`@@ -8,19 +8,21 @@ import (`
`8`	`8`	`)`
`9`	`9`
`10`	`10`	`func RegisterDefault(registry destregistry.Registry) error {`
`11`		`- aws, err := destaws.New()`
	`11`	`+ loader := registry.MetadataLoader()`
	`12`	`+`
	`13`	`+ aws, err := destaws.New(loader)`
`12`	`14`	`if err != nil {`
`13`	`15`	`return err`
`14`	`16`	`}`
`15`	`17`	`registry.RegisterProvider("aws", aws)`
`16`	`18`
`17`		`- rabbitmq, err := destrabbitmq.New()`
	`19`	`+ rabbitmq, err := destrabbitmq.New(loader)`
`18`	`20`	`if err != nil {`
`19`	`21`	`return err`
`20`	`22`	`}`
`21`	`23`	`registry.RegisterProvider("rabbitmq", rabbitmq)`
`22`	`24`
`23`		`- webhook, err := destwebhook.New()`
	`25`	`+ webhook, err := destwebhook.New(loader)`
`24`	`26`	`if err != nil {`
`25`	`27`	`return err`
`26`	`28`	`}`
Original file line number	Diff line number	Diff line change
`@@ -10,6 +10,7 @@ import (`
`10`	`10`	`"net/http"`
`11`	`11`
`12`	`12`	`"github.com/hookdeck/outpost/internal/destregistry"`
	`13`	`+ "github.com/hookdeck/outpost/internal/destregistry/metadata"`
`13`	`14`	`"github.com/hookdeck/outpost/internal/models"`
`14`	`15`	`)`
`15`	`16`
`@@ -23,8 +24,8 @@ type WebhookDestinationConfig struct {`
`23`	`24`
`24`	`25`	`var _ destregistry.Provider = (*WebhookDestination)(nil)`
`25`	`26`
`26`		`-func New() (*WebhookDestination, error) {`
`27`		`- base, err := destregistry.NewBaseProvider("webhook")`
	`27`	`+func New(loader metadata.MetadataLoader) (WebhookDestination, error) {`
	`28`	`+ base, err := destregistry.NewBaseProvider(loader, "webhook")`
`28`	`29`	`if err != nil {`
`29`	`30`	`return nil, err`
`30`	`31`	`}`