Skip to content

Commit 82a3fd3

Browse files
leggetterleggetter
authored
Merge pull request #441 from hookdeck/chore/comment-out-clickhouse
Comment out ClickHouse and remove from docs. Add Azure Service Bus config. Update config docs generation.
2 parents 9cc62ee + beb43fa commit 82a3fd3

File tree

11 files changed

+151
-142
lines changed

11 files changed

+151
-142
lines changed

‎cmd/configdocsgen/main.go‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -601,7 +601,11 @@ func generateYAMLPart(builder *strings.Builder, configInfo *ParsedConfig, allCon
601601

602602
// Required status as a comment
603603
if field.Required != "" && field.Required != "N" && field.Required != "false" {
604-
builder.WriteString(fmt.Sprintf("%s# Required: %s\n", indent, field.Required))
604+
requiredText := field.Required
605+
if field.Required == "C" {
606+
requiredText = "Conditional"
607+
}
608+
builder.WriteString(fmt.Sprintf("%s# Required: %s\n", indent, requiredText))
605609
}
606610

607611
// Field line

‎cmd/e2e/configs/basic.go‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -96,12 +96,12 @@ func setLogStorage(t *testing.T, c *config.Config, logStorage LogStorageType) er
9696
case LogStorageTypePostgres:
9797
postgresURL := testinfra.NewPostgresConfig(t)
9898
c.PostgresURL = postgresURL
99-
case LogStorageTypeClickHouse:
100-
clickHouseConfig := testinfra.NewClickHouseConfig(t)
101-
c.ClickHouse.Addr = clickHouseConfig.Addr
102-
c.ClickHouse.Username = clickHouseConfig.Username
103-
c.ClickHouse.Password = clickHouseConfig.Password
104-
c.ClickHouse.Database = clickHouseConfig.Database
99+
// case LogStorageTypeClickHouse:
100+
// clickHouseConfig := testinfra.NewClickHouseConfig(t)
101+
// c.ClickHouse.Addr = clickHouseConfig.Addr
102+
// c.ClickHouse.Username = clickHouseConfig.Username
103+
// c.ClickHouse.Password = clickHouseConfig.Password
104+
// c.ClickHouse.Database = clickHouseConfig.Database
105105
default:
106106
return fmt.Errorf("invalid log storage type: %s", logStorage)
107107
}

‎docs/pages/references/configuration.mdx‎

Lines changed: 46 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ Global configurations are provided through env variables or a YAML file. ConfigM
3232
| `AWS_SQS_SECRET_ACCESS_KEY` | AWS Secret Access Key for SQS. Required if AWS SQS is the chosen MQ provider. | `nil` | Conditional |
3333
| `AZURE_SERVICEBUS_CLIENT_ID` | Service principal client ID | `nil` | Yes |
3434
| `AZURE_SERVICEBUS_CLIENT_SECRET` | Service principal client secret | `nil` | Yes |
35+
| `AZURE_SERVICEBUS_CONNECTION_STRING` | Azure Service Bus connection string | `nil` | No |
3536
| `AZURE_SERVICEBUS_DELIVERY_SUBSCRIPTION` | Subscription name for delivery queue | `outpost-delivery-sub` | No |
3637
| `AZURE_SERVICEBUS_DELIVERY_TOPIC` | Topic name for delivery queue | `outpost-delivery` | No |
3738
| `AZURE_SERVICEBUS_LOG_SUBSCRIPTION` | Subscription name for log queue | `outpost-log-sub` | No |
@@ -40,10 +41,6 @@ Global configurations are provided through env variables or a YAML file. ConfigM
4041
| `AZURE_SERVICEBUS_RESOURCE_GROUP` | Azure resource group name | `nil` | Yes |
4142
| `AZURE_SERVICEBUS_SUBSCRIPTION_ID` | Azure subscription ID | `nil` | Yes |
4243
| `AZURE_SERVICEBUS_TENANT_ID` | Azure Active Directory tenant ID | `nil` | Yes |
43-
| `CLICKHOUSE_ADDR` | Address (host:port) of the ClickHouse server. Example: 'localhost:9000'. Required if ClickHouse is used for log storage. | `nil` | Conditional |
44-
| `CLICKHOUSE_DATABASE` | Database name in ClickHouse to use. | `outpost` | No |
45-
| `CLICKHOUSE_PASSWORD` | Password for ClickHouse authentication. | `nil` | No |
46-
| `CLICKHOUSE_USERNAME` | Username for ClickHouse authentication. | `nil` | No |
4744
| `DELIVERY_MAX_CONCURRENCY` | Maximum number of delivery attempts to process concurrently. | `1` | No |
4845
| `DELIVERY_TIMEOUT_SECONDS` | Timeout in seconds for HTTP requests made during event delivery to webhook destinations. | `5` | No |
4946
| `DESTINATIONS_AWS_KINESIS_METADATA_IN_PAYLOAD` | If true, includes Outpost metadata (event ID, topic, etc.) within the Kinesis record payload. | `true` | No |
@@ -92,6 +89,9 @@ Global configurations are provided through env variables or a YAML file. ConfigM
9289
| `PUBLISH_AWS_SQS_QUEUE` | Name of the SQS queue for publishing events. Required if AWS SQS is the chosen publish MQ provider. | `nil` | Conditional |
9390
| `PUBLISH_AWS_SQS_REGION` | AWS Region for the SQS publish queue. Required if AWS SQS is the chosen publish MQ provider. | `nil` | Conditional |
9491
| `PUBLISH_AWS_SQS_SECRET_ACCESS_KEY` | AWS Secret Access Key for the SQS publish queue. Required if AWS SQS is the chosen publish MQ provider. | `nil` | Conditional |
92+
| `PUBLISH_AZURE_SERVICEBUS_CONNECTION_STRING` | Azure Service Bus connection string for the publish queue. Required if Azure Service Bus is the chosen publish MQ provider. | `nil` | Conditional |
93+
| `PUBLISH_AZURE_SERVICEBUS_SUBSCRIPTION` | Name of the Azure Service Bus subscription to read published events from. Required if Azure Service Bus is the chosen publish MQ provider. | `nil` | Conditional |
94+
| `PUBLISH_AZURE_SERVICEBUS_TOPIC` | Name of the Azure Service Bus topic for publishing events. Required if Azure Service Bus is the chosen publish MQ provider. | `nil` | Conditional |
9595
| `PUBLISH_GCP_PUBSUB_PROJECT` | GCP Project ID for the Pub/Sub publish topic. Required if GCP Pub/Sub is the chosen publish MQ provider. | `nil` | Conditional |
9696
| `PUBLISH_GCP_PUBSUB_SERVICE_ACCOUNT_CREDENTIALS` | JSON string or path to a file containing GCP service account credentials for the Pub/Sub publish topic. Required if GCP Pub/Sub is chosen and not using implicit credentials. | `nil` | Conditional |
9797
| `PUBLISH_GCP_PUBSUB_SUBSCRIPTION` | Name of the GCP Pub/Sub subscription to read published events from. Required if GCP Pub/Sub is the chosen publish MQ provider. | `nil` | Conditional |
@@ -153,21 +153,6 @@ alert:
153153
# Enables or disables audit logging for significant events.
154154
audit_log: true
155155

156-
clickhouse:
157-
# Address (host:port) of the ClickHouse server. Example: 'localhost:9000'. Required if ClickHouse is used for log storage.
158-
# Required: C
159-
addr: ""
160-
161-
# Database name in ClickHouse to use.
162-
database: "outpost"
163-
164-
# Password for ClickHouse authentication.
165-
password: ""
166-
167-
# Username for ClickHouse authentication.
168-
username: ""
169-
170-
171156
# Maximum number of delivery attempts to process concurrently.
172157
delivery_max_concurrency: 1
173158

@@ -243,7 +228,7 @@ mqs:
243228
# Configuration for using AWS SQS as the message queue. Only one MQ provider should be configured.
244229
aws_sqs:
245230
# AWS Access Key ID for SQS. Required if AWS SQS is the chosen MQ provider.
246-
# Required: C
231+
# Required: Conditional
247232
access_key_id: ""
248233

249234
# Name of the SQS queue for delivery events.
@@ -256,11 +241,11 @@ mqs:
256241
log_queue: "outpost-log"
257242

258243
# AWS Region for SQS. Required if AWS SQS is the chosen MQ provider.
259-
# Required: C
244+
# Required: Conditional
260245
region: ""
261246

262247
# AWS Secret Access Key for SQS. Required if AWS SQS is the chosen MQ provider.
263-
# Required: C
248+
# Required: Conditional
264249
secret_access_key: ""
265250

266251

@@ -274,6 +259,9 @@ mqs:
274259
# Required: Y
275260
client_secret: ""
276261

262+
# Azure Service Bus connection string
263+
connection_string: ""
264+
277265
# Subscription name for delivery queue
278266
delivery_subscription: "outpost-delivery-sub"
279267

@@ -318,11 +306,11 @@ mqs:
318306
log_topic: "outpost-log"
319307

320308
# GCP Project ID for Pub/Sub. Required if GCP Pub/Sub is the chosen MQ provider.
321-
# Required: C
309+
# Required: Conditional
322310
project: ""
323311

324312
# JSON string or path to a file containing GCP service account credentials for Pub/Sub. Required if GCP Pub/Sub is the chosen MQ provider and not running in an environment with implicit credentials (e.g., GCE, GKE).
325-
# Required: C
313+
# Required: Conditional
326314
service_account_credentials: ""
327315

328316

@@ -338,7 +326,7 @@ mqs:
338326
log_queue: "outpost-log"
339327

340328
# RabbitMQ server connection URL (e.g., 'amqp://user:pass@host:port/vhost'). Required if RabbitMQ is the chosen MQ provider.
341-
# Required: C
329+
# Required: Conditional
342330
server_url: ""
343331

344332

@@ -350,22 +338,22 @@ otel:
350338
# OpenTelemetry configuration specific to logs.
351339
logs:
352340
# Specifies the OTLP exporter to use for this telemetry type (e.g., 'otlp'). Typically used with environment variables like OTEL_EXPORTER_OTLP_TRACES_ENDPOINT.
353-
# Required: C
341+
# Required: Conditional
354342
exporter: ""
355343

356344
# Specifies the OTLP protocol ('grpc' or 'http') for this telemetry type. Typically used with environment variables like OTEL_EXPORTER_OTLP_TRACES_PROTOCOL.
357-
# Required: C
345+
# Required: Conditional
358346
protocol: ""
359347

360348

361349
# OpenTelemetry configuration specific to metrics.
362350
metrics:
363351
# Specifies the OTLP exporter to use for this telemetry type (e.g., 'otlp'). Typically used with environment variables like OTEL_EXPORTER_OTLP_TRACES_ENDPOINT.
364-
# Required: C
352+
# Required: Conditional
365353
exporter: ""
366354

367355
# Specifies the OTLP protocol ('grpc' or 'http') for this telemetry type. Typically used with environment variables like OTEL_EXPORTER_OTLP_TRACES_PROTOCOL.
368-
# Required: C
356+
# Required: Conditional
369357
protocol: ""
370358

371359

@@ -375,11 +363,11 @@ otel:
375363
# OpenTelemetry configuration specific to traces.
376364
traces:
377365
# Specifies the OTLP exporter to use for this telemetry type (e.g., 'otlp'). Typically used with environment variables like OTEL_EXPORTER_OTLP_TRACES_ENDPOINT.
378-
# Required: C
366+
# Required: Conditional
379367
exporter: ""
380368

381369
# Specifies the OTLP protocol ('grpc' or 'http') for this telemetry type. Typically used with environment variables like OTEL_EXPORTER_OTLP_TRACES_PROTOCOL.
382-
# Required: C
370+
# Required: Conditional
383371
protocol: ""
384372

385373

@@ -413,53 +401,68 @@ portal:
413401
proxy_url: ""
414402

415403
# The expected Referer URL for accessing the portal. This is a security measure. Required if the Outpost Portal is enabled/used. Example: 'https://admin.example.com'.
416-
# Required: C
404+
# Required: Conditional
417405
referer_url: ""
418406

419407

420408
# Connection URL for PostgreSQL, used as an alternative log storage. Example: 'postgres://user:pass@host:port/dbname?sslmode=disable'. Required if ClickHouse is not configured and log storage is needed.
421-
# Required: C
409+
# Required: Conditional
422410
postgres: ""
423411

424412
publishmq:
425413
# Configuration for using AWS SQS as the publish message queue. Only one publish MQ provider should be configured.
426414
aws_sqs:
427415
# AWS Access Key ID for the SQS publish queue. Required if AWS SQS is the chosen publish MQ provider.
428-
# Required: C
416+
# Required: Conditional
429417
access_key_id: ""
430418

431419
# Custom AWS SQS endpoint URL for the publish queue. Optional.
432420
endpoint: ""
433421

434422
# Name of the SQS queue for publishing events. Required if AWS SQS is the chosen publish MQ provider.
435-
# Required: C
423+
# Required: Conditional
436424
queue: ""
437425

438426
# AWS Region for the SQS publish queue. Required if AWS SQS is the chosen publish MQ provider.
439-
# Required: C
427+
# Required: Conditional
440428
region: ""
441429

442430
# AWS Secret Access Key for the SQS publish queue. Required if AWS SQS is the chosen publish MQ provider.
443-
# Required: C
431+
# Required: Conditional
444432
secret_access_key: ""
445433

446434

435+
# Configuration for using Azure Service Bus as the publish message queue. Only one publish MQ provider should be configured.
436+
azure_servicebus:
437+
# Azure Service Bus connection string for the publish queue. Required if Azure Service Bus is the chosen publish MQ provider.
438+
# Required: Conditional
439+
connection_string: ""
440+
441+
# Name of the Azure Service Bus subscription to read published events from. Required if Azure Service Bus is the chosen publish MQ provider.
442+
# Required: Conditional
443+
subscription: ""
444+
445+
# Name of the Azure Service Bus topic for publishing events. Required if Azure Service Bus is the chosen publish MQ provider.
446+
# Required: Conditional
447+
topic: ""
448+
449+
447450
# Configuration for using GCP Pub/Sub as the publish message queue. Only one publish MQ provider should be configured.
448451
gcp_pubsub:
449452
# GCP Project ID for the Pub/Sub publish topic. Required if GCP Pub/Sub is the chosen publish MQ provider.
450-
# Required: C
453+
# Required: Conditional
451454
project: ""
452455

453456
# JSON string or path to a file containing GCP service account credentials for the Pub/Sub publish topic. Required if GCP Pub/Sub is chosen and not using implicit credentials.
454-
# Required: C
457+
# Required: Conditional
455458
service_account_credentials: ""
456459

457460
# Name of the GCP Pub/Sub subscription to read published events from. Required if GCP Pub/Sub is the chosen publish MQ provider.
458-
# Required: C
461+
# Required: Conditional
459462
subscription: ""
460463

461464
# Name of the GCP Pub/Sub topic for publishing events. Required if GCP Pub/Sub is the chosen publish MQ provider.
462-
# Required: C
465+
# Required: Conditional
463466
topic: ""
464467

465468

@@ -469,11 +472,11 @@ publishmq:
469472
exchange: ""
470473

471474
# Name of the RabbitMQ queue for publishing events. Required if RabbitMQ is the chosen publish MQ provider.
472-
# Required: C
475+
# Required: Conditional
473476
queue: ""
474477

475478
# RabbitMQ server connection URL for the publish queue. Required if RabbitMQ is the chosen publish MQ provider.
476-
# Required: C
479+
# Required: Conditional
477480
server_url: ""
478481

479482

‎internal/config/config.go‎

Lines changed: 34 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@ import (
77
"strings"
88

99
"github.com/caarlos0/env/v9"
10-
"github.com/hookdeck/outpost/internal/clickhouse"
1110
"github.com/hookdeck/outpost/internal/migrator"
1211
"github.com/hookdeck/outpost/internal/redis"
1312
"github.com/hookdeck/outpost/internal/telemetry"
@@ -59,10 +58,10 @@ type Config struct {
5958
HTTPUserAgent string `yaml:"http_user_agent" env:"HTTP_USER_AGENT" desc:"Custom HTTP User-Agent string for outgoing webhook deliveries. If unset, a default (OrganizationName/Version) is used." required:"N"`
6059

6160
// Infrastructure
62-
Redis RedisConfig `yaml:"redis"`
63-
ClickHouse ClickHouseConfig `yaml:"clickhouse"`
64-
PostgresURL string `yaml:"postgres" env:"POSTGRES_URL" desc:"Connection URL for PostgreSQL, used as an alternative log storage. Example: 'postgres://user:pass@host:port/dbname?sslmode=disable'. Required if ClickHouse is not configured and log storage is needed." required:"C"`
65-
MQs *MQsConfig `yaml:"mqs"`
61+
Redis RedisConfig `yaml:"redis"`
62+
// ClickHouse ClickHouseConfig `yaml:"clickhouse"`
63+
PostgresURL string `yaml:"postgres" env:"POSTGRES_URL" desc:"Connection URL for PostgreSQL, used as an alternative log storage. Example: 'postgres://user:pass@host:port/dbname?sslmode=disable'. Required if ClickHouse is not configured and log storage is needed." required:"C"`
64+
MQs *MQsConfig `yaml:"mqs"`
6665

6766
// PublishMQ
6867
PublishMQ PublishMQConfig `yaml:"publishmq"`
@@ -119,9 +118,9 @@ func (c *Config) InitDefaults() {
119118
Host: "127.0.0.1",
120119
Port: 6379,
121120
}
122-
c.ClickHouse = ClickHouseConfig{
123-
Database: "outpost",
124-
}
121+
// c.ClickHouse = ClickHouseConfig{
122+
// Database: "outpost",
123+
// }
125124
c.MQs = &MQsConfig{
126125
RabbitMQ: RabbitMQConfig{
127126
Exchange: "outpost",
@@ -328,24 +327,24 @@ func (c *RedisConfig) ToConfig() *redis.RedisConfig {
328327
}
329328
}
330329

331-
type ClickHouseConfig struct {
332-
Addr string `yaml:"addr" env:"CLICKHOUSE_ADDR" desc:"Address (host:port) of the ClickHouse server. Example: 'localhost:9000'. Required if ClickHouse is used for log storage." required:"C"`
333-
Username string `yaml:"username" env:"CLICKHOUSE_USERNAME" desc:"Username for ClickHouse authentication." required:"N"`
334-
Password string `yaml:"password" env:"CLICKHOUSE_PASSWORD" desc:"Password for ClickHouse authentication." required:"N"`
335-
Database string `yaml:"database" env:"CLICKHOUSE_DATABASE" desc:"Database name in ClickHouse to use." required:"N"`
336-
}
337-
338-
func (c *ClickHouseConfig) ToConfig() *clickhouse.ClickHouseConfig {
339-
if c.Addr == "" {
340-
return nil
341-
}
342-
return &clickhouse.ClickHouseConfig{
343-
Addr: c.Addr,
344-
Username: c.Username,
345-
Password: c.Password,
346-
Database: c.Database,
347-
}
348-
}
330+
// type ClickHouseConfig struct {
331+
// Addr string `yaml:"addr" env:"CLICKHOUSE_ADDR" desc:"Address (host:port) of the ClickHouse server. Example: 'localhost:9000'. Required if ClickHouse is used for log storage." required:"C"`
332+
// Username string `yaml:"username" env:"CLICKHOUSE_USERNAME" desc:"Username for ClickHouse authentication." required:"N"`
333+
// Password string `yaml:"password" env:"CLICKHOUSE_PASSWORD" desc:"Password for ClickHouse authentication." required:"N"`
334+
// Database string `yaml:"database" env:"CLICKHOUSE_DATABASE" desc:"Database name in ClickHouse to use." required:"N"`
335+
// }
336+
337+
// func (c *ClickHouseConfig) ToConfig() *clickhouse.ClickHouseConfig {
338+
// if c.Addr == "" {
339+
// return nil
340+
// }
341+
// return &clickhouse.ClickHouseConfig{
342+
// Addr: c.Addr,
343+
// Username: c.Username,
344+
// Password: c.Password,
345+
// Database: c.Database,
346+
// }
347+
// }
349348

350349
type AlertConfig struct {
351350
CallbackURL string `yaml:"callback_url" env:"ALERT_CALLBACK_URL" desc:"URL to which Outpost will send a POST request when an alert is triggered (e.g., for destination failures)." required:"N"`
@@ -381,9 +380,9 @@ func (c *Config) ToTelemetryApplicationInfo() telemetry.ApplicationInfo {
381380

382381
entityStore := "redis"
383382
logStore := "TODO"
384-
if c.ClickHouse.Addr != "" {
385-
logStore = "clickhouse"
386-
}
383+
// if c.ClickHouse.Addr != "" {
384+
// logStore = "clickhouse"
385+
// }
387386
if c.PostgresURL != "" {
388387
logStore = "postgres"
389388
}
@@ -404,11 +403,11 @@ func (c *Config) ToMigratorOpts() migrator.MigrationOpts {
404403
PG: migrator.MigrationOptsPG{
405404
URL: c.PostgresURL,
406405
},
407-
CH: migrator.MigrationOptsCH{
408-
Addr: c.ClickHouse.Addr,
409-
Username: c.ClickHouse.Username,
410-
Password: c.ClickHouse.Password,
411-
Database: c.ClickHouse.Database,
412-
},
406+
// CH: migrator.MigrationOptsCH{
407+
// Addr: c.ClickHouse.Addr,
408+
// Username: c.ClickHouse.Username,
409+
// Password: c.ClickHouse.Password,
410+
// Database: c.ClickHouse.Database,
411+
// },
413412
}
414413
}

‎internal/config/config_test.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -127,10 +127,10 @@ aes_encryption_secret: test-secret
127127
assert.Equal(t, 50, cfg.MaxDestinationsPerTenant)
128128
assert.Equal(t, 10, cfg.DeliveryTimeoutSeconds)
129129
assert.Equal(t, "test-secret", cfg.AESEncryptionSecret)
130-
assert.Equal(t, "localhost:9000", cfg.ClickHouse.Addr)
131-
assert.Equal(t, "default", cfg.ClickHouse.Username)
132-
assert.Equal(t, "secret", cfg.ClickHouse.Password)
133-
assert.Equal(t, "default", cfg.ClickHouse.Database)
130+
// assert.Equal(t, "localhost:9000", cfg.ClickHouse.Addr)
131+
// assert.Equal(t, "default", cfg.ClickHouse.Username)
132+
// assert.Equal(t, "secret", cfg.ClickHouse.Password)
133+
// assert.Equal(t, "default", cfg.ClickHouse.Database)
134134
}
135135

136136
func TestConfigFileResolution(t *testing.T) {

0 commit comments

Comments
 (0)