feat: Support config from YAML (#186)

* feat: Read config from YAML & env

* fix: Update services to use the new config

* chore: Remove unnecessary config

* chore: Remove unused config parser

* refactor: Config parsing logic

* chore: Config use OS interface for easier mock

* chore: Default config paths

* test: Config test suite

* chore: Improve config validation

* fix: Validation & config parsing & make sure services are running

* test: Extend e2e from default config

* refactor: Group destinations & portal config

* test: Mixed YAML & env

* fix: API_PORT

* fix: Support optional publishmq

* chore: Rename destination config env

Author: alexluong

cmd/e2e/configs/basic.go

@@ -8,69 +8,63 @@ import (
 	"github.com/google/uuid"
 	"github.com/hookdeck/outpost/internal/config"
 	"github.com/hookdeck/outpost/internal/infra"
-	"github.com/hookdeck/outpost/internal/mqs"
 	"github.com/hookdeck/outpost/internal/util/testinfra"
 	"github.com/hookdeck/outpost/internal/util/testutil"
 )
 
-func Basic(t *testing.T) *config.Config {
-	// Config
+func Basic(t *testing.T) config.Config {
+	// Get test infrastructure configs
 	redisConfig := testutil.CreateTestRedisConfig(t)
 	clickHouseConfig := testinfra.NewClickHouseConfig(t)
 	rabbitmqServerURL := testinfra.EnsureRabbitMQ()
-	deliveryMQConfig := mqs.QueueConfig{
-		RabbitMQ: &mqs.RabbitMQConfig{
-			ServerURL: rabbitmqServerURL,
-			Exchange:  uuid.New().String(),
-			Queue:     uuid.New().String(),
-		},
-		Policy: mqs.Policy{
-			RetryLimit: 5,
-		},
-	}
-	logMQConfig := mqs.QueueConfig{
-		RabbitMQ: &mqs.RabbitMQConfig{
-			ServerURL: rabbitmqServerURL,
-			Exchange:  uuid.New().String(),
-			Queue:     uuid.New().String(),
-		},
-		Policy: mqs.Policy{
-			RetryLimit: 5,
-		},
-	}
+
+	// Start with defaults
+	c := &config.Config{}
+	c.InitDefaults()
+
+	// Override only what's needed for e2e tests
+	c.Service = config.ServiceTypeSingular.String()
+	c.APIPort = testutil.RandomPortNumber()
+	c.APIKey = "apikey"
+	c.APIJWTSecret = "jwtsecret"
+	c.AESEncryptionSecret = "encryptionsecret"
+	c.Topics = testutil.TestTopics
+
+	// Infrastructure overrides
+	c.Redis.Host = redisConfig.Host
+	c.Redis.Port = redisConfig.Port
+	c.Redis.Password = redisConfig.Password
+	c.Redis.Database = redisConfig.Database
+
+	c.ClickHouse.Addr = clickHouseConfig.Addr
+	c.ClickHouse.Username = clickHouseConfig.Username
+	c.ClickHouse.Password = clickHouseConfig.Password
+	c.ClickHouse.Database = clickHouseConfig.Database
+
+	// MQ overrides
+	c.MQs.RabbitMQ.ServerURL = rabbitmqServerURL
+	c.MQs.RabbitMQ.Exchange = uuid.New().String()
+	c.MQs.RabbitMQ.DeliveryQueue = uuid.New().String()
+	c.MQs.RabbitMQ.LogQueue = uuid.New().String()
+
+	// Test-specific overrides
+	c.PublishMaxConcurrency = 3
+	c.DeliveryMaxConcurrency = 3
+	c.LogMaxConcurrency = 3
+	c.RetryIntervalSeconds = 1
+	c.RetryMaxLimit = 3
+	c.LogBatcherDelayThresholdSeconds = 1
+	c.LogBatcherItemCountThreshold = 100
+
+	// Setup cleanup
 	t.Cleanup(func() {
 		if err := infra.Teardown(context.Background(), infra.Config{
-			DeliveryMQ: &deliveryMQConfig,
-			LogMQ:      &logMQConfig,
+			DeliveryMQ: c.MQs.GetDeliveryQueueConfig(),
+			LogMQ:      c.MQs.GetLogQueueConfig(),
 		}); err != nil {
 			log.Println("Teardown failed:", err)
 		}
 	})
 
-	return &config.Config{
-		Hostname:                        "outpost",
-		Service:                         config.ServiceTypeSingular,
-		Port:                            testutil.RandomPortNumber(),
-		APIKey:                          "apikey",
-		APIJWTSecret:                    "jwtsecret",
-		AESEncryptionSecret:             "encryptionsecret",
-		PortalProxyURL:                  "",
-		Topics:                          testutil.TestTopics,
-		Redis:                           redisConfig,
-		ClickHouse:                      &clickHouseConfig,
-		OpenTelemetry:                   nil,
-		PublishQueueConfig:              nil,
-		DeliveryQueueConfig:             &deliveryMQConfig,
-		LogQueueConfig:                  &logMQConfig,
-		PublishMaxConcurrency:           3,
-		DeliveryMaxConcurrency:          3,
-		LogMaxConcurrency:               3,
-		RetryIntervalSeconds:            1,
-		RetryMaxLimit:                   3,
-		DeliveryTimeoutSeconds:          5,
-		LogBatcherDelayThresholdSeconds: 1,
-		LogBatcherItemCountThreshold:    100,
-		MaxDestinationsPerTenant:        20,
-		DestinationWebhookHeaderPrefix:  "x-outpost-",
-	}
+	return *c
 }

cmd/e2e/suites_test.go

@@ -22,17 +22,17 @@ import (
 
 type e2eSuite struct {
 	ctx               context.Context
-	config            *config.Config
+	config            config.Config
 	mockServerBaseURL string
 	mockServerInfra   *testinfra.MockServerInfra
 	cleanup           func()
 	client            httpclient.Client
 }
 
 func (suite *e2eSuite) SetupSuite() {
-	suite.client = httpclient.New(fmt.Sprintf("http://localhost:%d/api/v1", suite.config.Port), suite.config.APIKey)
+	suite.client = httpclient.New(fmt.Sprintf("http://localhost:%d/api/v1", suite.config.APIPort), suite.config.APIKey)
 	go func() {
-		application := app.New(suite.config)
+		application := app.New(&suite.config)
 		if err := application.Run(suite.ctx); err != nil {
 			log.Println("Application failed to run", err)
 		}
@@ -122,9 +122,13 @@ func (suite *basicSuite) SetupSuite() {
 	t.Cleanup(testinfra.Start(t))
 	gin.SetMode(gin.TestMode)
 	mockServerBaseURL := testinfra.GetMockServer(t)
+
+	cfg := configs.Basic(t)
+	require.NoError(t, cfg.Validate(config.Flags{}))
+
 	suite.e2eSuite = e2eSuite{
 		ctx:               context.Background(),
-		config:            configs.Basic(t),
+		config:            cfg,
 		mockServerBaseURL: mockServerBaseURL,
 		mockServerInfra:   testinfra.NewMockServerInfra(mockServerBaseURL),
 		cleanup:           func() {},

go.mod

@@ -12,6 +12,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/credentials v1.17.27
 	github.com/aws/aws-sdk-go-v2/service/sqs v1.34.3
 	github.com/aws/smithy-go v1.20.3
+	github.com/caarlos0/env/v9 v9.0.0
 	github.com/gin-contrib/static v1.1.2
 	github.com/gin-gonic/gin v1.10.0
 	github.com/go-playground/validator/v10 v10.22.0
@@ -55,6 +56,7 @@ require (
 	gocloud.dev v0.39.0
 	gocloud.dev/pubsub/rabbitpubsub v0.39.0
 	golang.org/x/sync v0.8.0
+	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
@@ -183,5 +185,4 @@ require (
 	google.golang.org/grpc v1.65.0 // indirect
 	google.golang.org/protobuf v1.34.2 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
-	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -76,6 +76,8 @@ github.com/bytedance/sonic v1.11.9 h1:LFHENlIY/SLzDWverzdOvgMztTxcfcF+cqNsz9pK5z
 github.com/bytedance/sonic v1.11.9/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4=
 github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM=
 github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU=
+github.com/caarlos0/env/v9 v9.0.0 h1:SI6JNsOA+y5gj9njpgybykATIylrRMklbs5ch6wO6pc=
+github.com/caarlos0/env/v9 v9.0.0/go.mod h1:ye5mlCVMYh6tZ+vCgrs/B95sj88cg5Tlnc0XIzgZ020=
 github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
 github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=

internal/app/app.go

@@ -43,7 +43,7 @@ func run(mainContext context.Context, cfg *config.Config) error {
 		otelzap.WithMinLevel(zap.InfoLevel), // TODO: allow configuration
 	)
 
-	chDB, err := clickhouse.New(cfg.ClickHouse)
+	chDB, err := clickhouse.New(cfg.ClickHouse.ToConfig())
 	if err != nil {
 		return err
 	}
@@ -53,8 +53,8 @@ func run(mainContext context.Context, cfg *config.Config) error {
 	}
 
 	if err := infra.Declare(mainContext, infra.Config{
-		DeliveryMQ: cfg.DeliveryQueueConfig,
-		LogMQ:      cfg.LogQueueConfig,
+		DeliveryMQ: cfg.MQs.GetDeliveryQueueConfig(),
+		LogMQ:      cfg.MQs.GetLogQueueConfig(),
 	}); err != nil {
 		return err
 	}
@@ -63,8 +63,8 @@ func run(mainContext context.Context, cfg *config.Config) error {
 	ctx, cancel := context.WithCancel(mainContext)
 
 	// Set up OpenTelemetry.
-	if cfg.OpenTelemetry != nil {
-		otelShutdown, err := otel.SetupOTelSDK(ctx, cfg.OpenTelemetry)
+	if cfg.OpenTelemetry.ToConfig() != nil {
+		otelShutdown, err := otel.SetupOTelSDK(ctx, cfg.OpenTelemetry.ToConfig())
 		if err != nil {
 			cancel()
 			return err
@@ -121,23 +121,24 @@ func constructServices(
 	wg *sync.WaitGroup,
 	logger *otelzap.Logger,
 ) ([]Service, error) {
+	serviceType := cfg.MustGetService()
 	services := []Service{}
 
-	if cfg.Service == config.ServiceTypeAPI || cfg.Service == config.ServiceTypeSingular {
+	if serviceType == config.ServiceTypeAPI || serviceType == config.ServiceTypeSingular {
 		service, err := api.NewService(ctx, wg, cfg, logger)
 		if err != nil {
 			return nil, err
 		}
 		services = append(services, service)
 	}
-	if cfg.Service == config.ServiceTypeDelivery || cfg.Service == config.ServiceTypeSingular {
+	if serviceType == config.ServiceTypeDelivery || serviceType == config.ServiceTypeSingular {
 		service, err := delivery.NewService(ctx, wg, cfg, logger, nil)
 		if err != nil {
 			return nil, err
 		}
 		services = append(services, service)
 	}
-	if cfg.Service == config.ServiceTypeLog || cfg.Service == config.ServiceTypeSingular {
+	if serviceType == config.ServiceTypeLog || serviceType == config.ServiceTypeSingular {
 		service, err := log.NewService(ctx, wg, cfg, logger, nil)
 		if err != nil {
 			return nil, err