feat(clickhouse): add s3_tier storage policy with renamed disks

- Add `s3_tier` storage policy: local NVMe as hot tier, S3 as cold tier
- Rename disks for clarity: `default` → `local`, `s3_cache` → `s3`
- Add `--s3-tier-move-factor` option to `clickhouse init` (default: 0.2)
- Store `s3TierMoveFactor` in ClickHouseConfig and wire through manifest builder
- Inject `CLICKHOUSE_S3_TIER_MOVE_FACTOR` env var into ClickHouse pods
- Update Event.ClickHouse.ConfigSaved to include s3TierMoveFactor
- Add end-to-end test: creates table with s3_tier policy, inserts data,
  forces move to S3, verifies bucket is non-empty

Closes #582

Co-authored-by: Jon Haddad <rustyrazorblade@users.noreply.github.com>

Author: github-actions[bot]

bin/end-to-end-test

@@ -892,6 +892,51 @@ EOF
     clickhouse-query "INSERT INTO test (id) VALUES (1)"
 }
 
+step_clickhouse_s3_tier_test() {
+    if [ "$ENABLE_CLICKHOUSE" != true ]; then
+        echo "=== Skipping ClickHouse S3 tier test (use --clickhouse to enable) ==="
+        return 0
+    fi
+    echo "=== Testing ClickHouse S3 tier storage policy ==="
+
+    local data_bucket
+    data_bucket=$(jq -r '.dataBucket // empty' state.json 2>/dev/null)
+    if [ -z "$data_bucket" ]; then
+        echo "ERROR: dataBucket not found in state.json"
+        return 1
+    fi
+    echo "Data bucket: $data_bucket"
+
+    echo "--- Creating table with s3_tier storage policy ---"
+    clickhouse-query <<'EOF'
+CREATE OR REPLACE TABLE test_s3_tier (
+    id UInt64,
+    ts DateTime DEFAULT now()
+) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/default/test_s3_tier', '{replica}')
+ORDER BY id
+SETTINGS storage_policy = 's3_tier'
+EOF
+
+    echo "--- Inserting 10000 rows ---"
+    clickhouse-query "INSERT INTO test_s3_tier (id) SELECT number FROM numbers(10000)"
+
+    echo "--- Forcing data move to S3 disk ---"
+    clickhouse-query "ALTER TABLE test_s3_tier MOVE PARTITION tuple() TO DISK 's3'"
+
+    echo "--- Verifying data exists in S3 bucket ---"
+    local s3_object_count
+    s3_object_count=$(aws s3 ls "s3://${data_bucket}/clickhouse/" --recursive 2>/dev/null | wc -l)
+    echo "S3 object count under clickhouse/: $s3_object_count"
+    if [ "$s3_object_count" -gt 0 ]; then
+        echo "S3 tier test PASSED: $s3_object_count objects found in s3://${data_bucket}/clickhouse/"
+    else
+        echo "ERROR: No objects found in s3://${data_bucket}/clickhouse/ after moving data to S3 tier"
+        return 1
+    fi
+
+    echo "=== ClickHouse S3 tier test completed ==="
+}
+
 step_clickhouse_stop() {
     if [ "$ENABLE_CLICKHOUSE" != true ]; then
         echo "=== Skipping ClickHouse stop (use --clickhouse to enable) ==="
@@ -1245,6 +1290,7 @@ STEPS_WORKDIR=(
     "step_cassandra_start_stop:Cassandra start/stop cycle"
     "step_clickhouse_start:Start ClickHouse"
     "step_clickhouse_test:Test ClickHouse"
+    "step_clickhouse_s3_tier_test:Test ClickHouse S3 tier policy"
     "step_clickhouse_stop:Stop ClickHouse"
     "step_opensearch_start:Start OpenSearch"
     "step_opensearch_test:Test OpenSearch"

openspec/changes/clickhouse-s3-tier/design.md

@@ -0,0 +1,63 @@
+# Design: ClickHouse S3 Tier Storage Policy
+
+## Data Flow
+
+```
+clickhouse init --s3-tier-move-factor 0.1
+  → ClickHouseConfig.s3TierMoveFactor = 0.1
+  → saved to state.json
+
+clickhouse start
+  → ClickHouseManifestBuilder.buildClusterConfigMap(s3TierMoveFactor = 0.1)
+      → ConfigMap key: "s3-tier-move-factor" = "0.1"
+  → ClickHouseManifestBuilder.buildServerContainer()
+      → env var: CLICKHOUSE_S3_TIER_MOVE_FACTOR from ConfigMap
+  → Pod reads env var → config.xml reads <move_factor from_env="CLICKHOUSE_S3_TIER_MOVE_FACTOR"/>
+```
+
+## config.xml Changes
+
+### Disk renames
+- `default` → `local` (explicit `type=local`, path `/mnt/db1/clickhouse/`)
+- `s3_cache` → `s3` (cache layer over `s3_disk`, path `/mnt/db1/clickhouse/disks/s3/`)
+
+### New policy
+```xml
+<s3_tier>
+    <volumes>
+        <hot><disk>local</disk></hot>
+        <cold><disk>s3</disk></cold>
+    </volumes>
+    <move_factor from_env="CLICKHOUSE_S3_TIER_MOVE_FACTOR"/>
+</s3_tier>
+```
+
+## Kotlin Changes
+
+### Constants.ClickHouse
+```kotlin
+const val DEFAULT_S3_TIER_MOVE_FACTOR = 0.2
+```
+
+### ClickHouseConfig (ClusterState.kt)
+```kotlin
+data class ClickHouseConfig(
+    ...
+    val s3TierMoveFactor: Double = Constants.ClickHouse.DEFAULT_S3_TIER_MOVE_FACTOR,
+)
+```
+
+### ClickHouseInit
+New option: `--s3-tier-move-factor`
+
+### ClickHouseManifestBuilder
+- `buildClusterConfigMap`: adds `"s3-tier-move-factor"` key
+- `buildServerContainer`: adds `CLICKHOUSE_S3_TIER_MOVE_FACTOR` env var from ConfigMap
+- `buildAllResources` + `buildClusterConfigMap`: accept `s3TierMoveFactor` param
+- `buildServerInitDataDirContainer`: creates `/mnt/db1/clickhouse/disks/s3` (renamed from `s3_cache`)
+
+### ClickHouseStart
+Passes `clickHouseConfig.s3TierMoveFactor` to `buildAllResources`.
+
+### Event.ClickHouse.ConfigSaved
+New field: `s3TierMoveFactor: Double`

openspec/changes/clickhouse-s3-tier/proposal.md

@@ -0,0 +1,41 @@
+# Proposal: ClickHouse S3 Tier Storage Policy
+
+## Problem
+
+ClickHouse supports two storage modes today:
+- `local`: NVMe only
+- `s3_main`: S3 with local cache (data lives primarily in S3)
+
+There is no policy for a tiered approach where data starts on fast local NVMe and migrates to S3 only when local disk fills up. This is the classic "hot/cold" tiering pattern that maximises write speed while keeping storage costs low.
+
+## Proposed Solution
+
+Add a new `s3_tier` storage policy to ClickHouse with:
+- **Hot volume**: local NVMe disk (`local` disk)
+- **Cold volume**: S3 with local cache (`s3` disk)
+- **Automatic migration**: controlled by `move_factor` — when local disk is `(1 - move_factor)` full, ClickHouse moves the oldest parts to the cold volume
+
+The move factor is configurable via `--s3-tier-move-factor` on `clickhouse init` (default: `0.2`, meaning data moves when local disk reaches 80% capacity).
+
+## Disk Rename
+
+As part of this change, disk names are made more descriptive:
+- `default` → `local` (explicit local disk definition)
+- `s3_cache` → `s3` (the cache-over-S3 disk)
+
+This makes storage policies self-documenting: `local` policy uses `local` disk, `s3_main` policy uses `s3` disk, `s3_tier` policy uses both.
+
+## User Experience
+
+```bash
+# Use default move factor (0.2)
+easy-db-lab clickhouse init
+
+# Use custom move factor
+easy-db-lab clickhouse init --s3-tier-move-factor 0.1
+
+easy-db-lab clickhouse start
+
+# Create a table with tiered storage
+clickhouse-query "CREATE TABLE events ... SETTINGS storage_policy = 's3_tier'"
+```

openspec/changes/clickhouse-s3-tier/tasks.md

@@ -0,0 +1,26 @@
+# Tasks: ClickHouse S3 Tier Storage Policy
+
+## Implementation
+
+- [x] Add `DEFAULT_S3_TIER_MOVE_FACTOR = 0.2` to `Constants.ClickHouse`
+- [x] Add `s3TierMoveFactor: Double` field to `ClickHouseConfig`
+- [x] Add `--s3-tier-move-factor` option to `ClickHouseInit`
+- [x] Update `Event.ClickHouse.ConfigSaved` to include `s3TierMoveFactor`
+- [x] Rename disks in `config.xml`: `default` → `local`, `s3_cache` → `s3`
+- [x] Add `s3_tier` policy to `config.xml`
+- [x] Update `ClickHouseManifestBuilder.buildClusterConfigMap` to store `s3-tier-move-factor`
+- [x] Update `ClickHouseManifestBuilder.buildServerContainer` to inject `CLICKHOUSE_S3_TIER_MOVE_FACTOR` env var
+- [x] Update `ClickHouseManifestBuilder.buildAllResources` signature
+- [x] Update `buildServerInitDataDirContainer` to create `/mnt/db1/clickhouse/disks/s3`
+- [x] Wire `ClickHouseStart` to pass `s3TierMoveFactor` from config to builder
+
+## Tests
+
+- [x] Update `ClickHouseManifestBuilderTest` to pass `s3TierMoveFactor`
+- [x] Add assertion for `s3-tier-move-factor` in cluster config ConfigMap test
+- [x] Add test asserting `s3_tier` policy appears in `config.xml`
+- [x] Add `step_clickhouse_s3_tier_test` to `bin/end-to-end-test`:
+  - Creates table with `s3_tier` policy
+  - Inserts 10,000 rows
+  - Forces move to S3 disk via `ALTER TABLE ... MOVE PARTITION tuple() TO DISK 's3'`
+  - Verifies S3 bucket contains objects under `clickhouse/` prefix

src/main/kotlin/com/rustyrazorblade/easydblab/Constants.kt

@@ -237,6 +237,7 @@ object Constants {
         const val DEFAULT_S3_CACHE_SIZE = "10Gi"
         const val DEFAULT_S3_CACHE_ON_WRITE = "true"
         const val DEFAULT_REPLICAS_PER_SHARD = 3
+        const val DEFAULT_S3_TIER_MOVE_FACTOR = 0.2
     }
 
     // YACE (Yet Another CloudWatch Exporter) configuration

src/main/kotlin/com/rustyrazorblade/easydblab/commands/clickhouse/ClickHouseInit.kt

@@ -40,13 +40,20 @@ class ClickHouseInit : PicoBaseCommand() {
     )
     var replicasPerShard: Int = Constants.ClickHouse.DEFAULT_REPLICAS_PER_SHARD
 
+    @Option(
+        names = ["--s3-tier-move-factor"],
+        description = ["Fraction of local disk free space that triggers data move to S3 tier (default: \${DEFAULT-VALUE})"],
+    )
+    var s3TierMoveFactor: Double = Constants.ClickHouse.DEFAULT_S3_TIER_MOVE_FACTOR
+
     override fun execute() {
         val state = clusterStateManager.load()
         val config =
             ClickHouseConfig(
                 s3CacheSize = s3CacheSize,
                 s3CacheOnWrite = s3CacheOnWrite,
                 replicasPerShard = replicasPerShard,
+                s3TierMoveFactor = s3TierMoveFactor,
             )
         state.updateClickHouseConfig(config)
         clusterStateManager.save(state)
@@ -56,6 +63,7 @@ class ClickHouseInit : PicoBaseCommand() {
                 replicasPerShard = replicasPerShard,
                 s3CacheSize = s3CacheSize,
                 s3CacheOnWrite = s3CacheOnWrite,
+                s3TierMoveFactor = s3TierMoveFactor,
             ),
         )
     }

src/main/kotlin/com/rustyrazorblade/easydblab/commands/clickhouse/ClickHouseStart.kt

@@ -26,6 +26,7 @@ import picocli.CommandLine.Option
  * Storage policies available for tables:
  * - 'local': Local disk storage (default)
  * - 's3_main': S3 storage with local cache
+ * - 's3_tier': Local NVMe as hot tier, S3 as cold tier (automatic data migration)
  *
  * Example creating a distributed replicated table:
  * ```sql
@@ -197,6 +198,7 @@ class ClickHouseStart : PicoBaseCommand() {
                 replicasPerShard = replicasPerShard,
                 s3CacheSize = clickHouseConfig.s3CacheSize,
                 s3CacheOnWrite = clickHouseConfig.s3CacheOnWrite,
+                s3TierMoveFactor = clickHouseConfig.s3TierMoveFactor,
             )
 
         for (resource in resources) {

src/main/kotlin/com/rustyrazorblade/easydblab/configuration/ClusterState.kt

@@ -156,6 +156,7 @@ data class ClickHouseConfig(
     val s3CacheSize: String = Constants.ClickHouse.DEFAULT_S3_CACHE_SIZE,
     val s3CacheOnWrite: String = Constants.ClickHouse.DEFAULT_S3_CACHE_ON_WRITE,
     val replicasPerShard: Int = Constants.ClickHouse.DEFAULT_REPLICAS_PER_SHARD,
+    val s3TierMoveFactor: Double = Constants.ClickHouse.DEFAULT_S3_TIER_MOVE_FACTOR,
 )
 
 /**

src/main/kotlin/com/rustyrazorblade/easydblab/configuration/clickhouse/ClickHouseManifestBuilder.kt

@@ -80,18 +80,20 @@ class ClickHouseManifestBuilder(
      * @param replicasPerShard Number of replicas per shard
      * @param s3CacheSize S3 cache size (e.g. "10Gi")
      * @param s3CacheOnWrite Whether to cache on write operations
+     * @param s3TierMoveFactor Fraction of local disk free space that triggers move to S3 tier
      * @return List of all K8s resources
      */
     fun buildAllResources(
         totalReplicas: Int,
         replicasPerShard: Int,
         s3CacheSize: String,
         s3CacheOnWrite: String,
+        s3TierMoveFactor: Double = Constants.ClickHouse.DEFAULT_S3_TIER_MOVE_FACTOR,
     ): List<HasMetadata> =
         listOf(
             buildKeeperConfigMap(),
             buildServerConfigMap(totalReplicas, replicasPerShard),
-            buildClusterConfigMap(replicasPerShard, s3CacheSize, s3CacheOnWrite),
+            buildClusterConfigMap(replicasPerShard, s3CacheSize, s3CacheOnWrite, s3TierMoveFactor),
             buildKeeperService(),
             buildServerHeadlessService(),
             buildServerClientService(),
@@ -143,6 +145,7 @@ class ClickHouseManifestBuilder(
         replicasPerShard: Int,
         s3CacheSize: String,
         s3CacheOnWrite: String,
+        s3TierMoveFactor: Double = Constants.ClickHouse.DEFAULT_S3_TIER_MOVE_FACTOR,
     ): HasMetadata =
         ConfigMapBuilder()
             .withNewMetadata()
@@ -153,6 +156,7 @@ class ClickHouseManifestBuilder(
             .addToData("replicas-per-shard", replicasPerShard.toString())
             .addToData("s3-cache-size", s3CacheSize)
             .addToData("s3-cache-on-write", s3CacheOnWrite)
+            .addToData("s3-tier-move-factor", s3TierMoveFactor.toString())
             .build()
 
     /**
@@ -518,7 +522,7 @@ class ClickHouseManifestBuilder(
                 mkdir -p /mnt/db1/clickhouse/user_files
                 mkdir -p /mnt/db1/clickhouse/format_schemas
                 mkdir -p /mnt/db1/clickhouse/disks/s3_disk
-                mkdir -p /mnt/db1/clickhouse/disks/s3_cache
+                mkdir -p /mnt/db1/clickhouse/disks/s3
                 mkdir -p /mnt/db1/clickhouse/logs
                 chown -R 101:101 /mnt/db1/clickhouse
                 """.trimIndent(),
@@ -596,6 +600,17 @@ class ClickHouseManifestBuilder(
                                         .build(),
                                 ).build(),
                         ).build(),
+                    EnvVarBuilder()
+                        .withName("CLICKHOUSE_S3_TIER_MOVE_FACTOR")
+                        .withValueFrom(
+                            EnvVarSourceBuilder()
+                                .withConfigMapKeyRef(
+                                    ConfigMapKeySelectorBuilder()
+                                        .withName("clickhouse-cluster-config")
+                                        .withKey("s3-tier-move-factor")
+                                        .build(),
+                                ).build(),
+                        ).build(),
                 ),
             ).withEnvFrom(
                 io.fabric8.kubernetes.api.model

src/main/kotlin/com/rustyrazorblade/easydblab/events/Event.kt

@@ -4462,13 +4462,15 @@ sealed interface Event {
             val replicasPerShard: Int,
             val s3CacheSize: String,
             val s3CacheOnWrite: String,
+            val s3TierMoveFactor: Double,
         ) : ClickHouse {
             override fun toDisplayString(): String =
                 """
                 ClickHouse configuration saved.
                   Replicas per shard: $replicasPerShard
                   S3 cache size: $s3CacheSize
                   S3 cache on write: $s3CacheOnWrite
+                  S3 tier move factor: $s3TierMoveFactor
                 """.trimIndent()
         }
     }