feat(metactl): add comprehensive test suite for all subcommands (#18431)

- Add `get` and `trigger-snapshot` subcommands to metactl CLI
- Implement complete test coverage for all metactl subcommands:
  status, upsert, get, watch, trigger-snapshot, export-from-grpc,
  export-from-raft-dir, import, transfer-leader
- Extract shared utilities into metactl_utils.py and utils.py
- Add unified test runner (test_all_subcommands.py) for CI integration
- Add comprehensive documentation (README_SUBCOMMAND_TESTS.md)

Tests include precise verification of command outputs, error handling,
and end-to-end functionality validation with proper cleanup.

Author: drmingdrmer

.github/actions/test_metactl/action.yml

@@ -16,4 +16,5 @@ runs:
       shell: bash
       run: |
         python -m pip install -r ./tests/metactl/requirements.txt
+        python ./tests/metactl/test_all_subcommands.py
         python ./tests/metactl/test-metactl-restore-new-cluster.py

src/meta/binaries/metactl/main.rs

@@ -28,12 +28,14 @@ use databend_common_meta_client::MetaGrpcClient;
 use databend_common_meta_control::admin::MetaAdminClient;
 use databend_common_meta_control::args::BenchArgs;
 use databend_common_meta_control::args::ExportArgs;
+use databend_common_meta_control::args::GetArgs;
 use databend_common_meta_control::args::GlobalArgs;
 use databend_common_meta_control::args::ImportArgs;
 use databend_common_meta_control::args::ListFeatures;
 use databend_common_meta_control::args::SetFeature;
 use databend_common_meta_control::args::StatusArgs;
 use databend_common_meta_control::args::TransferLeaderArgs;
+use databend_common_meta_control::args::TriggerSnapshotArgs;
 use databend_common_meta_control::args::UpsertArgs;
 use databend_common_meta_control::args::WatchArgs;
 use databend_common_meta_control::export_from_disk;
@@ -162,6 +164,13 @@ impl App {
         Ok(())
     }
 
+    async fn trigger_snapshot(&self, args: &TriggerSnapshotArgs) -> anyhow::Result<()> {
+        let client = MetaAdminClient::new(args.admin_api_address.as_str());
+        client.trigger_snapshot().await?;
+        println!("triggered snapshot successfully.");
+        Ok(())
+    }
+
     async fn export(&self, args: &ExportArgs) -> anyhow::Result<()> {
         match args.raft_dir {
             None => {
@@ -208,7 +217,20 @@ impl App {
         Ok(())
     }
 
+    async fn get(&self, args: &GetArgs) -> anyhow::Result<()> {
+        let addresses = vec![args.grpc_api_address.clone()];
+        let client = self.new_grpc_client(addresses)?;
+
+        let res = client.get_kv(&args.key).await?;
+        println!("{}", serde_json::to_string(&res)?);
+        Ok(())
+    }
+
     fn new_grpc_client(&self, addresses: Vec<String>) -> Result<Arc<ClientHandle>, CreationError> {
+        eprintln!(
+            "Using gRPC API address: {}",
+            serde_json::to_string(&addresses).unwrap()
+        );
         MetaGrpcClient::try_create(
             addresses,
             "root",
@@ -226,11 +248,13 @@ enum CtlCommand {
     Export(ExportArgs),
     Import(ImportArgs),
     TransferLeader(TransferLeaderArgs),
+    TriggerSnapshot(TriggerSnapshotArgs),
     SetFeature(SetFeature),
     ListFeatures(ListFeatures),
     BenchClientNumConn(BenchArgs),
     Watch(WatchArgs),
     Upsert(UpsertArgs),
+    Get(GetArgs),
 }
 
 /// Usage:
@@ -271,6 +295,9 @@ async fn main() -> anyhow::Result<()> {
             CtlCommand::TransferLeader(args) => {
                 app.transfer_leader(args).await?;
             }
+            CtlCommand::TriggerSnapshot(args) => {
+                app.trigger_snapshot(args).await?;
+            }
             CtlCommand::SetFeature(args) => {
                 let client = MetaAdminClient::new(args.admin_api_address.as_str());
                 let res = client.set_feature(&args.feature, args.enable).await?;
@@ -296,6 +323,9 @@ async fn main() -> anyhow::Result<()> {
             CtlCommand::Upsert(args) => {
                 app.upsert(args).await?;
             }
+            CtlCommand::Get(args) => {
+                app.get(args).await?;
+            }
         },
         // for backward compatibility
         None => {

src/meta/control/src/admin.rs

@@ -79,6 +79,22 @@ impl MetaAdminClient {
         }
     }
 
+    pub async fn trigger_snapshot(&self) -> anyhow::Result<()> {
+        let resp = self
+            .client
+            .get(format!("{}/v1/ctrl/trigger_snapshot", self.endpoint))
+            .send()
+            .await?;
+        let status = resp.status();
+        if status.is_success() {
+            Ok(())
+        } else {
+            let data = resp.bytes().await?;
+            let msg = String::from_utf8_lossy(&data);
+            Err(anyhow::anyhow!("status code: {}, msg: {}", status, msg))
+        }
+    }
+
     pub async fn set_feature(
         &self,
         feature: &str,

src/meta/control/src/args.rs

@@ -247,3 +247,19 @@ pub struct UpsertArgs {
     #[clap(long)]
     pub value: String,
 }
+
+#[derive(Debug, Clone, Deserialize, Args)]
+pub struct GetArgs {
+    #[clap(long, default_value = "127.0.0.1:9191")]
+    pub grpc_api_address: String,
+
+    /// The key to get
+    #[clap(long)]
+    pub key: String,
+}
+
+#[derive(Debug, Clone, Deserialize, Args)]
+pub struct TriggerSnapshotArgs {
+    #[clap(long, default_value = "127.0.0.1:28101")]
+    pub admin_api_address: String,
+}

tests/metactl/README_SUBCOMMAND_TESTS.md

@@ -0,0 +1,56 @@
+# Metactl Subcommand Tests
+
+Comprehensive tests for all `databend-metactl` subcommands.
+
+## Usage
+
+### Build Prerequisites
+```bash
+cargo build --bin databend-metactl --bin databend-meta
+```
+
+### Run Tests
+```bash
+# All tests
+python tests/metactl/test_all_subcommands.py
+
+# Individual tests
+python tests/metactl/subcommands/<name>.py
+```
+
+## Test Structure
+
+```
+tests/metactl/
+├── test_all_subcommands.py          # Main test runner
+├── subcommands/                     # Subcommand test modules
+│   ├── __init__.py
+│   └── <name>.py                    # Individual subcommand tests
+├── metactl_utils.py                 # Metactl utilities
+├── utils.py                         # General utilities
+├── config/                          # Node configurations
+└── data files                       # Test data
+```
+
+## Test Execution Order
+
+1. Basic: `status`, `upsert`, `get`
+2. Data operations: `watch`, `trigger-snapshot`
+3. Export/Import: `export-from-grpc`, `export-from-raft-dir`, `import`
+4. Advanced: `transfer-leader`
+
+## CI Integration
+
+```yaml
+test-metactl-subcommands:
+  script:
+    - cargo build --bin databend-metactl --bin databend-meta
+    - python tests/metactl/test_all_subcommands.py
+```
+
+## Guidelines
+
+- Use global imports: `from utils import`
+- Cleanup only on test success
+- Check actual results, not just command success
+- Follow naming: `subcommands/<name>.py`

tests/metactl/metactl_utils.py

@@ -0,0 +1,128 @@
+#!/usr/bin/env python3
+
+import json
+import subprocess
+import time
+from typing import Dict
+import requests
+from pathlib import Path
+from utils import print_step, BUILD_PROFILE, run_command
+
+metactl_bin = f"./target/{BUILD_PROFILE}/databend-metactl"
+
+
+def metactl_upsert(grpc_addr, key, value):
+    """Upsert a key-value pair using the upsert subcommand."""
+    result = run_command([
+        metactl_bin, "upsert",
+        "--grpc-api-address", grpc_addr,
+        "--key", key,
+        "--value", value
+    ])
+    return result
+
+
+def metactl_trigger_snapshot(admin_addr):
+    """Trigger snapshot creation using the trigger-snapshot subcommand."""
+    result = run_command([
+        metactl_bin, "trigger-snapshot",
+        "--admin-api-address", admin_addr
+    ])
+    return result
+
+
+def verify_kv(grpc_addr, key, expected_value=None):
+    """Verify a key-value pair using the get subcommand."""
+    time.sleep(0.5)  # Brief sleep to ensure data is persisted
+
+    result = run_command([
+        metactl_bin, "get",
+        "--grpc-api-address", grpc_addr,
+        "--key", key
+    ], check=False)
+
+    print(f"Get result for key '{key}': {result}")
+
+    if not result.strip():
+        assert False, f"Key '{key}' not found or empty result"
+
+    try:
+        data = json.loads(result.strip())
+    except json.JSONDecodeError as e:
+        assert False, f"Failed to parse JSON result: {e}, result: {result}"
+
+    print(f"Parsed data: {data}")
+
+    if data is None:
+        assert expected_value is None, f"Expected None but got {expected_value}"
+    else:
+        # Extract the actual value from the stored data
+        actual_value = bytes(data["data"]).decode('utf-8')
+        print(f"Actual value: '{actual_value}', Expected: '{expected_value}'")
+
+        if expected_value is not None:
+            assert actual_value == expected_value, f"Expected '{expected_value}', got '{actual_value}'"
+
+
+def metactl_export_from_grpc(addr: str) -> str:
+    """Export meta data from grpc-address to stdout"""
+    print_step(f"Start: Export meta data from {addr} to stdout")
+
+    cmd = [metactl_bin, "export", "--grpc-api-address", addr]
+
+    result = run_command(cmd, check=True)
+
+    print_step(f"Done: Exported meta data to stdout")
+    return result
+
+def metactl_export(meta_dir: str, output_path: str) -> str:
+    """Export meta data from raft directory to database file or stdout"""
+    print_step(f"Start: Export meta data from {meta_dir} to {output_path}")
+
+    cmd = [metactl_bin, "export", "--raft-dir", meta_dir]
+
+    if output_path:
+        cmd.append("--db")
+        cmd.append(output_path)
+
+    result = run_command(cmd, check=False)
+
+    print_step(f"Done: Exported meta data to {output_path}")
+    return result
+
+
+def metactl_import(
+    meta_dir: str, id: int, db_path: str, initial_cluster: Dict[int, str]
+):
+    """Import meta data from database file to raft directory"""
+    cmd = [
+        metactl_bin,
+        "import",
+        "--raft-dir",
+        meta_dir,
+        "--id",
+        str(id),
+        "--db",
+        db_path,
+    ]
+
+    for id, addr in initial_cluster.items():
+        cmd.append("--initial-cluster")
+        cmd.append(f"{id}={addr}")
+
+    result = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    result.wait()
+    print(result.stdout.read().decode())
+    print(result.stderr.read().decode())
+
+
+def cluster_status(msg: str) -> str:
+    """Get cluster status from meta service endpoint"""
+    print_step(f"Check /v1/cluster/status {msg} start")
+
+    response = requests.get("http://127.0.0.1:28101/v1/cluster/status")
+    status = response.json()
+
+    print_step(f"Check /v1/cluster/status {msg} end")
+
+    return status

tests/metactl/subcommands/__init__.py

@@ -0,0 +1 @@
+# Subcommand tests package