Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions dev/k8s/manifests/vector-logs.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,7 @@ data:
.project_id = .kubernetes.pod_labels."unkey.com/project.id" || ""
.environment_id = .kubernetes.pod_labels."unkey.com/environment.id" || ""
.deployment_id = .kubernetes.pod_labels."unkey.com/deployment.id" || ""
.app_id = .kubernetes.pod_labels."unkey.com/app.id" || ""
# Keep k8s metadata
.k8s_pod_name = .kubernetes.pod_name || ""
Expand Down
32 changes: 32 additions & 0 deletions pkg/clickhouse/migrations/20260220000000.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
-- Breaking change: drop and recreate runtime_logs_raw_v1 to add app_id to ORDER BY.
-- ClickHouse doesn't allow altering the ORDER BY of a MergeTree table.
-- All existing log data will be lost — acceptable since logs are ephemeral.

DROP TABLE IF EXISTS default.runtime_logs_raw_v1;

CREATE TABLE IF NOT EXISTS default.runtime_logs_raw_v1
(
`time` Int64 CODEC(Delta, LZ4),
`inserted_at` Int64 DEFAULT toUnixTimestamp64Milli(now64(3)),
`severity` LowCardinality(String),
`message` String CODEC(ZSTD(1)),
`workspace_id` String CODEC(ZSTD(1)),
`project_id` String CODEC(ZSTD(1)),
`environment_id` String CODEC(ZSTD(1)),
`app_id` String DEFAULT '' CODEC(ZSTD(1)),
`deployment_id` String CODEC(ZSTD(1)),
`k8s_pod_name` String CODEC(ZSTD(1)),
`region` LowCardinality(String),
`attributes` JSON CODEC(ZSTD(1)),
`attributes_text` String MATERIALIZED toJSONString(attributes) CODEC(ZSTD(1)),
`expires_at` DateTime64(3) DEFAULT now64(3) + INTERVAL 90 DAY,
INDEX idx_workspace_id workspace_id TYPE bloom_filter(0.001) GRANULARITY 1,
INDEX idx_deployment_id deployment_id TYPE bloom_filter(0.001) GRANULARITY 1,
INDEX idx_message message TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 1,
INDEX idx_attributes_text attributes_text TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 1
)
ENGINE = MergeTree()
PARTITION BY toDate(fromUnixTimestamp64Milli(inserted_at))
ORDER BY (workspace_id, project_id, environment_id, app_id, time, deployment_id)
TTL expires_at + INTERVAL 7 DAY
SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1;
3 changes: 2 additions & 1 deletion pkg/clickhouse/schema/023_runtime_logs_raw_v1.sql
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ CREATE TABLE IF NOT EXISTS default.runtime_logs_raw_v1
`workspace_id` String CODEC(ZSTD(1)),
`project_id` String CODEC(ZSTD(1)),
`environment_id` String CODEC(ZSTD(1)),
`app_id` String DEFAULT '' CODEC(ZSTD(1)),
`deployment_id` String CODEC(ZSTD(1)),

-- K8s metadata (pod name for identifying specific replica)
Expand All @@ -43,6 +44,6 @@ CREATE TABLE IF NOT EXISTS default.runtime_logs_raw_v1
)
ENGINE = MergeTree()
PARTITION BY toDate(fromUnixTimestamp64Milli(inserted_at))
ORDER BY (workspace_id, project_id, environment_id, time, deployment_id)
ORDER BY (workspace_id, project_id, environment_id, app_id, time, deployment_id)
TTL expires_at + INTERVAL 7 DAY
SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1;
16 changes: 16 additions & 0 deletions pkg/db/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -25,12 +25,28 @@ go_library(
"api_insert.sql_generated.go",
"api_soft_delete.sql_generated.go",
"api_update_delete_protection.sql_generated.go",
"app_build_settings_find.sql_generated.go",
"app_build_settings_upsert.sql_generated.go",
"app_environment_variables_find.sql_generated.go",
"app_find_by_id.sql_generated.go",
"app_find_by_project_and_slug.sql_generated.go",
"app_find_with_settings.sql_generated.go",
"app_insert.sql_generated.go",
"app_internal_service_upsert.sql_generated.go",
"app_list_by_project.sql_generated.go",
"app_runtime_settings_find.sql_generated.go",
"app_runtime_settings_upsert.sql_generated.go",
"app_update_deployments.sql_generated.go",
"audit_log_find_target_by_id.sql_generated.go",
"audit_log_insert.sql_generated.go",
"audit_log_target_insert.sql_generated.go",
"bulk_acme_challenge_insert.sql_generated.go",
"bulk_acme_user_insert.sql_generated.go",
"bulk_api_insert.sql_generated.go",
"bulk_app_build_settings_upsert.sql_generated.go",
"bulk_app_insert.sql_generated.go",
"bulk_app_internal_service_upsert.sql_generated.go",
"bulk_app_runtime_settings_upsert.sql_generated.go",
"bulk_audit_log_insert.sql_generated.go",
"bulk_audit_log_target_insert.sql_generated.go",
"bulk_certificate_insert.sql_generated.go",
Expand Down
48 changes: 48 additions & 0 deletions pkg/db/app_build_settings_find.sql_generated.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

81 changes: 81 additions & 0 deletions pkg/db/app_build_settings_upsert.sql_generated.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

56 changes: 56 additions & 0 deletions pkg/db/app_environment_variables_find.sql_generated.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

45 changes: 45 additions & 0 deletions pkg/db/app_find_by_id.sql_generated.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

52 changes: 52 additions & 0 deletions pkg/db/app_find_by_project_and_slug.sql_generated.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading
Loading