feat(clickhouse): add s3_tier storage policy with automatic tiering

Add a new s3_tier storage policy that provides automatic data movement
from local disks to S3 based on disk space availability. This combines
local performance for hot data with S3's unlimited capacity for cold data.

Key features:
- Starts with local NVMe storage for best write performance
- Automatically moves oldest data to S3 when disk free space falls below
  configured threshold (default: 20%)
- Configurable via --s3-tier-move-factor (range: 0.0-1.0)
- Data on S3 remains queryable with cache-assisted reads

Implementation:
- Add s3_tier policy with local and s3 disk volumes to storage.xml
- Add --s3-tier-move-factor option to clickhouse init command
- Include input validation to reject values outside [0.0, 1.0]
- Update ClickHouse config generation to use renamed disk identifiers
  (local_nvme, s3) for consistency across all policies
- Add comprehensive unit tests for boundary validation
- Add E2E test verifying data movement to S3

Documentation:
- Update docs/user-guide/clickhouse.md with s3_tier policy guide
- Add policy comparison table including all three policies
- Document when to use tiered storage and how it works

Author: rustyrazorblade

bin/end-to-end-test

@@ -892,6 +892,54 @@ EOF
     clickhouse-query "INSERT INTO test (id) VALUES (1)"
 }
 
+step_clickhouse_s3_tier_test() {
+    if [ "$ENABLE_CLICKHOUSE" != true ]; then
+        echo "=== Skipping ClickHouse S3 tier test (use --clickhouse to enable) ==="
+        return 0
+    fi
+    echo "=== Testing ClickHouse S3 tier storage policy ==="
+
+    local data_bucket
+    data_bucket=$(jq -r '.dataBucket // empty' state.json 2>/dev/null)
+    if [ -z "$data_bucket" ]; then
+        echo "ERROR: dataBucket not found in state.json"
+        return 1
+    fi
+    echo "Data bucket: $data_bucket"
+
+    echo "--- Creating table with s3_tier storage policy ---"
+    clickhouse-query <<'EOF'
+CREATE OR REPLACE TABLE test_s3_tier (
+    id UInt64,
+    ts DateTime DEFAULT now()
+) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/default/test_s3_tier', '{replica}')
+ORDER BY id
+SETTINGS storage_policy = 's3_tier'
+EOF
+
+    echo "--- Inserting 10000 rows ---"
+    clickhouse-query "INSERT INTO test_s3_tier (id) SELECT number FROM numbers(10000)"
+
+    echo "--- Forcing data move to S3 disk ---"
+    clickhouse-query "ALTER TABLE test_s3_tier MOVE PARTITION tuple() TO DISK 's3'"
+
+    echo "--- Verifying data exists in S3 bucket ---"
+    local s3_object_count
+    s3_object_count=$(aws s3 ls "s3://${data_bucket}/clickhouse/" --recursive 2>/dev/null | wc -l)
+    echo "S3 object count under clickhouse/: $s3_object_count"
+    if [ "$s3_object_count" -gt 0 ]; then
+        echo "S3 tier test PASSED: $s3_object_count objects found in s3://${data_bucket}/clickhouse/"
+    else
+        echo "ERROR: No objects found in s3://${data_bucket}/clickhouse/ after moving data to S3 tier"
+        return 1
+    fi
+
+    echo "--- Cleaning up test table ---"
+    clickhouse-query "DROP TABLE IF EXISTS test_s3_tier"
+
+    echo "=== ClickHouse S3 tier test completed ==="
+}
+
 step_clickhouse_stop() {
     if [ "$ENABLE_CLICKHOUSE" != true ]; then
         echo "=== Skipping ClickHouse stop (use --clickhouse to enable) ==="
@@ -1245,6 +1293,7 @@ STEPS_WORKDIR=(
     "step_cassandra_start_stop:Cassandra start/stop cycle"
     "step_clickhouse_start:Start ClickHouse"
     "step_clickhouse_test:Test ClickHouse"
+    "step_clickhouse_s3_tier_test:Test ClickHouse S3 tier policy"
     "step_clickhouse_stop:Stop ClickHouse"
     "step_opensearch_start:Start OpenSearch"
     "step_opensearch_test:Test OpenSearch"

docs/user-guide/clickhouse.md

@@ -34,6 +34,8 @@ easy-db-lab clickhouse init --s3-cache-on-write false
 |--------|-------------|---------|
 | `--s3-cache` | Size of the local S3 cache | 10Gi |
 | `--s3-cache-on-write` | Cache data during write operations | true |
+| `--s3-tier-move-factor` | Move data to S3 tier when local disk free space falls below this fraction (0.0-1.0) | 0.2 |
+| `--replicas-per-shard` | Number of replicas per shard | 3 |
 
 Configuration is saved to the cluster state and applied when you run `clickhouse start`.
 
@@ -189,14 +191,14 @@ ClickHouse is configured with two storage policies. You select the policy when c
 
 ### Policy Comparison
 
-| Aspect | `local` | `s3_main` |
-|--------|---------|-----------|
-| **Storage Location** | Local NVMe disks | S3 bucket with configurable local cache |
-| **Performance** | Best latency, highest throughput | Higher latency, cache-dependent |
-| **Capacity** | Limited by disk size | Virtually unlimited |
-| **Cost** | Included in instance cost | S3 storage + request costs |
-| **Data Persistence** | Lost when cluster is destroyed | Persists independently |
-| **Best For** | Benchmarks, low-latency queries | Large datasets, cost-sensitive workloads |
+| Aspect | `local` | `s3_main` | `s3_tier` |
+|--------|---------|-----------|-----------|
+| **Storage Location** | Local NVMe disks | S3 bucket with configurable local cache | Hybrid: starts local, moves to S3 when disk fills |
+| **Performance** | Best latency, highest throughput | Higher latency, cache-dependent | Good initially, degrades as data moves to S3 |
+| **Capacity** | Limited by disk size | Virtually unlimited | Virtually unlimited |
+| **Cost** | Included in instance cost | S3 storage + request costs | S3 storage + request costs |
+| **Data Persistence** | Lost when cluster is destroyed | Persists independently | Persists independently |
+| **Best For** | Benchmarks, low-latency queries | Large datasets, cost-sensitive workloads | Mixed hot/cold workloads with automatic tiering |
 
 ### Local Storage (`local`)
 
@@ -251,6 +253,50 @@ SETTINGS storage_policy = 's3_main';
 - Cache is automatically managed by ClickHouse
 - First query on cold data will be slower; subsequent queries use cache
 
+### S3 Tiered Storage (`s3_tier`)
+
+The S3 tiered policy provides automatic data movement from local disks to S3 based on disk space availability. This policy starts with local storage and automatically moves data to S3 when local disk space runs low, providing the best of both worlds: fast local performance for hot data and unlimited S3 capacity for cold data.
+
+**Prerequisite**: Your cluster must be initialized with an S3 bucket. Set this during `init`:
+
+```bash
+easy-db-lab init my-cluster --s3-bucket my-clickhouse-data
+```
+
+Configure the tiering behavior before starting ClickHouse:
+
+```bash
+# Move data to S3 when local disk free space falls below 20% (default)
+easy-db-lab clickhouse init --s3-tier-move-factor 0.2
+
+# More aggressive tiering - move when free space < 50%
+easy-db-lab clickhouse init --s3-tier-move-factor 0.5
+```
+
+Then create tables with S3 tiered storage:
+
+```sql
+CREATE TABLE my_table (...)
+ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/default/my_table', '{replica}')
+ORDER BY id
+SETTINGS storage_policy = 's3_tier';
+```
+
+**When to use S3 tiered storage:**
+
+- Workloads with mixed hot/cold data access patterns
+- Growing datasets that may outgrow local disk capacity
+- Want automatic cost optimization without manual intervention
+- Need local performance for recent data with S3 capacity for historical data
+
+**How automatic tiering works:**
+
+- New data is written to local disks first (fast writes)
+- When local disk free space falls below the configured threshold (default: 20%), ClickHouse automatically moves the oldest data to S3
+- Data on S3 is still queryable but with higher latency
+- The local cache (configured with `--s3-cache`) helps performance for frequently accessed S3 data
+- Manual moves are also possible: `ALTER TABLE my_table MOVE PARTITION tuple() TO DISK 's3'`
+
 ## Stopping ClickHouse
 
 To remove the ClickHouse cluster:

openspec/changes/clickhouse-s3-tier/design.md

@@ -0,0 +1,63 @@
+# Design: ClickHouse S3 Tier Storage Policy
+
+## Data Flow
+
+```
+clickhouse init --s3-tier-move-factor 0.1
+  → ClickHouseConfig.s3TierMoveFactor = 0.1
+  → saved to state.json
+
+clickhouse start
+  → ClickHouseManifestBuilder.buildClusterConfigMap(s3TierMoveFactor = 0.1)
+      → ConfigMap key: "s3-tier-move-factor" = "0.1"
+  → ClickHouseManifestBuilder.buildServerContainer()
+      → env var: CLICKHOUSE_S3_TIER_MOVE_FACTOR from ConfigMap
+  → Pod reads env var → config.xml reads <move_factor from_env="CLICKHOUSE_S3_TIER_MOVE_FACTOR"/>
+```
+
+## config.xml Changes
+
+### Disk renames
+- `default` → `local` (explicit `type=local`, path `/mnt/db1/clickhouse/`)
+- `s3_cache` → `s3` (cache layer over `s3_disk`, path `/mnt/db1/clickhouse/disks/s3/`)
+
+### New policy
+```xml
+<s3_tier>
+    <volumes>
+        <hot><disk>local</disk></hot>
+        <cold><disk>s3</disk></cold>
+    </volumes>
+    <move_factor from_env="CLICKHOUSE_S3_TIER_MOVE_FACTOR"/>
+</s3_tier>
+```
+
+## Kotlin Changes
+
+### Constants.ClickHouse
+```kotlin
+const val DEFAULT_S3_TIER_MOVE_FACTOR = 0.2
+```
+
+### ClickHouseConfig (ClusterState.kt)
+```kotlin
+data class ClickHouseConfig(
+    ...
+    val s3TierMoveFactor: Double = Constants.ClickHouse.DEFAULT_S3_TIER_MOVE_FACTOR,
+)
+```
+
+### ClickHouseInit
+New option: `--s3-tier-move-factor`
+
+### ClickHouseManifestBuilder
+- `buildClusterConfigMap`: adds `"s3-tier-move-factor"` key
+- `buildServerContainer`: adds `CLICKHOUSE_S3_TIER_MOVE_FACTOR` env var from ConfigMap
+- `buildAllResources` + `buildClusterConfigMap`: accept `s3TierMoveFactor` param
+- `buildServerInitDataDirContainer`: creates `/mnt/db1/clickhouse/disks/s3` (renamed from `s3_cache`)
+
+### ClickHouseStart
+Passes `clickHouseConfig.s3TierMoveFactor` to `buildAllResources`.
+
+### Event.ClickHouse.ConfigSaved
+New field: `s3TierMoveFactor: Double`

openspec/changes/clickhouse-s3-tier/proposal.md

@@ -0,0 +1,41 @@
+# Proposal: ClickHouse S3 Tier Storage Policy
+
+## Problem
+
+ClickHouse supports two storage modes today:
+- `local`: NVMe only
+- `s3_main`: S3 with local cache (data lives primarily in S3)
+
+There is no policy for a tiered approach where data starts on fast local NVMe and migrates to S3 only when local disk fills up. This is the classic "hot/cold" tiering pattern that maximises write speed while keeping storage costs low.
+
+## Proposed Solution
+
+Add a new `s3_tier` storage policy to ClickHouse with:
+- **Hot volume**: local NVMe disk (`local` disk)
+- **Cold volume**: S3 with local cache (`s3` disk)
+- **Automatic migration**: controlled by `move_factor` — when local disk is `(1 - move_factor)` full, ClickHouse moves the oldest parts to the cold volume
+
+The move factor is configurable via `--s3-tier-move-factor` on `clickhouse init` (default: `0.2`, meaning data moves when local disk reaches 80% capacity).
+
+## Disk Rename
+
+As part of this change, disk names are made more descriptive:
+- `default` → `local` (explicit local disk definition)
+- `s3_cache` → `s3` (the cache-over-S3 disk)
+
+This makes storage policies self-documenting: `local` policy uses `local` disk, `s3_main` policy uses `s3` disk, `s3_tier` policy uses both.
+
+## User Experience
+
+```bash
+# Use default move factor (0.2)
+easy-db-lab clickhouse init
+
+# Use custom move factor
+easy-db-lab clickhouse init --s3-tier-move-factor 0.1
+
+easy-db-lab clickhouse start
+
+# Create a table with tiered storage
+clickhouse-query "CREATE TABLE events ... SETTINGS storage_policy = 's3_tier'"
+```

openspec/changes/clickhouse-s3-tier/tasks.md

@@ -0,0 +1,26 @@
+# Tasks: ClickHouse S3 Tier Storage Policy
+
+## Implementation
+
+- [x] Add `DEFAULT_S3_TIER_MOVE_FACTOR = 0.2` to `Constants.ClickHouse`
+- [x] Add `s3TierMoveFactor: Double` field to `ClickHouseConfig`
+- [x] Add `--s3-tier-move-factor` option to `ClickHouseInit`
+- [x] Update `Event.ClickHouse.ConfigSaved` to include `s3TierMoveFactor`
+- [x] Rename disks in `config.xml`: `default` → `local`, `s3_cache` → `s3`
+- [x] Add `s3_tier` policy to `config.xml`
+- [x] Update `ClickHouseManifestBuilder.buildClusterConfigMap` to store `s3-tier-move-factor`
+- [x] Update `ClickHouseManifestBuilder.buildServerContainer` to inject `CLICKHOUSE_S3_TIER_MOVE_FACTOR` env var
+- [x] Update `ClickHouseManifestBuilder.buildAllResources` signature
+- [x] Update `buildServerInitDataDirContainer` to create `/mnt/db1/clickhouse/disks/s3`
+- [x] Wire `ClickHouseStart` to pass `s3TierMoveFactor` from config to builder
+
+## Tests
+
+- [x] Update `ClickHouseManifestBuilderTest` to pass `s3TierMoveFactor`
+- [x] Add assertion for `s3-tier-move-factor` in cluster config ConfigMap test
+- [x] Add test asserting `s3_tier` policy appears in `config.xml`
+- [x] Add `step_clickhouse_s3_tier_test` to `bin/end-to-end-test`:
+  - Creates table with `s3_tier` policy
+  - Inserts 10,000 rows
+  - Forces move to S3 disk via `ALTER TABLE ... MOVE PARTITION tuple() TO DISK 's3'`
+  - Verifies S3 bucket contains objects under `clickhouse/` prefix

src/main/kotlin/com/rustyrazorblade/easydblab/Constants.kt

@@ -237,6 +237,7 @@ object Constants {
         const val DEFAULT_S3_CACHE_SIZE = "10Gi"
         const val DEFAULT_S3_CACHE_ON_WRITE = "true"
         const val DEFAULT_REPLICAS_PER_SHARD = 3
+        const val DEFAULT_S3_TIER_MOVE_FACTOR = 0.2
     }
 
     // YACE (Yet Another CloudWatch Exporter) configuration

src/main/kotlin/com/rustyrazorblade/easydblab/commands/clickhouse/ClickHouseInit.kt

@@ -40,13 +40,25 @@ class ClickHouseInit : PicoBaseCommand() {
     )
     var replicasPerShard: Int = Constants.ClickHouse.DEFAULT_REPLICAS_PER_SHARD
 
+    @Option(
+        names = ["--s3-tier-move-factor"],
+        description = ["Move data to S3 tier when local disk free space falls below this fraction (0.0-1.0) (default: \${DEFAULT-VALUE})"],
+    )
+    var s3TierMoveFactor: Double = Constants.ClickHouse.DEFAULT_S3_TIER_MOVE_FACTOR
+
     override fun execute() {
+        // Validate s3TierMoveFactor range [0.0, 1.0] per ClickHouse requirements
+        require(s3TierMoveFactor in 0.0..1.0) {
+            "s3TierMoveFactor must be in range [0.0, 1.0], got: $s3TierMoveFactor"
+        }
+
         val state = clusterStateManager.load()
         val config =
             ClickHouseConfig(
                 s3CacheSize = s3CacheSize,
                 s3CacheOnWrite = s3CacheOnWrite,
                 replicasPerShard = replicasPerShard,
+                s3TierMoveFactor = s3TierMoveFactor,
             )
         state.updateClickHouseConfig(config)
         clusterStateManager.save(state)
@@ -56,6 +68,7 @@ class ClickHouseInit : PicoBaseCommand() {
                 replicasPerShard = replicasPerShard,
                 s3CacheSize = s3CacheSize,
                 s3CacheOnWrite = s3CacheOnWrite,
+                s3TierMoveFactor = s3TierMoveFactor,
             ),
         )
     }

src/main/kotlin/com/rustyrazorblade/easydblab/commands/clickhouse/ClickHouseStart.kt

@@ -26,6 +26,7 @@ import picocli.CommandLine.Option
  * Storage policies available for tables:
  * - 'local': Local disk storage (default)
  * - 's3_main': S3 storage with local cache
+ * - 's3_tier': Local NVMe as hot tier, S3 as cold tier (automatic data migration)
  *
  * Example creating a distributed replicated table:
  * ```sql
@@ -197,6 +198,7 @@ class ClickHouseStart : PicoBaseCommand() {
                 replicasPerShard = replicasPerShard,
                 s3CacheSize = clickHouseConfig.s3CacheSize,
                 s3CacheOnWrite = clickHouseConfig.s3CacheOnWrite,
+                s3TierMoveFactor = clickHouseConfig.s3TierMoveFactor,
             )
 
         for (resource in resources) {

src/main/kotlin/com/rustyrazorblade/easydblab/configuration/ClusterState.kt

@@ -156,6 +156,7 @@ data class ClickHouseConfig(
     val s3CacheSize: String = Constants.ClickHouse.DEFAULT_S3_CACHE_SIZE,
     val s3CacheOnWrite: String = Constants.ClickHouse.DEFAULT_S3_CACHE_ON_WRITE,
     val replicasPerShard: Int = Constants.ClickHouse.DEFAULT_REPLICAS_PER_SHARD,
+    val s3TierMoveFactor: Double = Constants.ClickHouse.DEFAULT_S3_TIER_MOVE_FACTOR,
 )
 
 /**