diff --git a/cmd/deploy.go b/cmd/deploy.go index 4dba5bae84..c87571f50a 100644 --- a/cmd/deploy.go +++ b/cmd/deploy.go @@ -129,6 +129,23 @@ func deployCmd(o *Options) (*cobra.Command, error) { //nolint: funlen "lab owner name (only for users in clab_admins group)", ) + c.Flags().StringVar( + &o.Deploy.RestoreAll, + "restore-all", + "", + "restore all nodes that have snapshots in this directory (default: ./snapshots)", + ) + // Allow flag without value to default to ./snapshots + c.Flags().Lookup("restore-all").NoOptDefVal = "./snapshots" + + c.Flags().StringArrayVar( + &o.Deploy.RestoreNodeSnapshots, + "restore", + nil, + "restore specific node from snapshot file (format: node=path/to/snapshot.tar). "+ + "Can be specified multiple times. Overrides --restore-all for specified nodes.", + ) + return c, nil } @@ -157,7 +174,9 @@ func deployFn(cobraCmd *cobra.Command, o *Options) error { SetReconfigure(o.Deploy.Reconfigure). SetGraph(o.Deploy.GenerateGraph). SetSkipPostDeploy(o.Deploy.SkipPostDeploy). - SetSkipLabDirFileACLs(o.Deploy.SkipLabDirectoryFileACLs) + SetSkipLabDirFileACLs(o.Deploy.SkipLabDirectoryFileACLs). + SetRestoreAll(o.Deploy.RestoreAll). + SetRestoreNodeSnapshots(o.Deploy.RestoreNodeSnapshots) containers, err := c.Deploy(cobraCmd.Context(), deploymentOptions) if err != nil { diff --git a/cmd/options.go b/cmd/options.go index d77814afdc..f3a259c0ba 100644 --- a/cmd/options.go +++ b/cmd/options.go @@ -102,6 +102,12 @@ func GetOptions() *Options { SrcPort: 0, DeletionPrefix: "vx-", }, + ToolsSnapshot: &ToolsSnapshotOptions{ + OutputDir: "./snapshots", + Format: "table", + Timeout: 5 * time.Minute, + MaxConcurrent: 0, + }, Version: &VersionOptions{ Short: false, JSON: false, @@ -129,6 +135,7 @@ type Options struct { ToolsSSHX *ToolsSSHXOptions ToolsVeth *ToolsVethOptions ToolsVxlan *ToolsVxlanOptions + ToolsSnapshot *ToolsSnapshotOptions Version *VersionOptions } @@ -276,6 +283,8 @@ type DeployOptions struct { SkipLabDirectoryFileACLs bool ExportTemplate string LabOwner string + RestoreAll string + RestoreNodeSnapshots []string } func (o *DeployOptions) toClabOptions() []clabcore.ClabOption { @@ -443,6 +452,13 @@ type ToolsVxlanOptions struct { DeletionPrefix string } +type ToolsSnapshotOptions struct { + OutputDir string + Format string + Timeout time.Duration + MaxConcurrent int +} + type VersionOptions struct { Short bool JSON bool diff --git a/cmd/tools.go b/cmd/tools.go index c52a53856e..d52c063763 100644 --- a/cmd/tools.go +++ b/cmd/tools.go @@ -19,6 +19,7 @@ func toolsSubcommandRegisterFuncs() []func(*Options) (*cobra.Command, error) { disableTxOffloadCmd, gottyCmd, netemCmd, + snapshotCmd, sshxCmd, vethCmd, vxlanCmd, diff --git a/cmd/tools_snapshot.go b/cmd/tools_snapshot.go new file mode 100644 index 0000000000..46c0fc11b4 --- /dev/null +++ b/cmd/tools_snapshot.go @@ -0,0 +1,385 @@ +// Copyright 2020 Nokia +// Licensed under the BSD 3-Clause License. +// SPDX-License-Identifier: BSD-3-Clause + +package cmd + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/charmbracelet/log" + "github.com/spf13/cobra" + clabconstants "github.com/srl-labs/containerlab/constants" + clabcore "github.com/srl-labs/containerlab/core" + clabexec "github.com/srl-labs/containerlab/exec" + clabruntime "github.com/srl-labs/containerlab/runtime" +) + +const ( + defaultSnapshotTimeout = 5 * time.Minute + snapshotPollInterval = 2 * time.Second +) + +func snapshotCmd(o *Options) (*cobra.Command, error) { + c := &cobra.Command{ + Use: "snapshot", + Short: "snapshot operations for vrnetlab-based nodes", + Long: "snapshot command provides operations to save and manage VM snapshots " + + "for vrnetlab-based nodes in your lab", + } + + saveCmd := &cobra.Command{ + Use: "save", + Short: "save VM snapshots from running nodes", + Long: "save creates snapshots of running vrnetlab-based VMs and saves them to disk.\n" + + "Each node's snapshot is saved as {output-dir}/{nodename}.tar\n" + + "Non-vrnetlab nodes are automatically skipped.", + RunE: func(cmd *cobra.Command, _ []string) error { + return snapshotSaveFn(cmd, o) + }, + } + + saveCmd.Flags().StringVar( + &o.ToolsSnapshot.OutputDir, + "output-dir", + o.ToolsSnapshot.OutputDir, + "directory to save snapshot files (creates {dir}/{node}.tar)", + ) + + saveCmd.Flags().StringSliceVar( + &o.Filter.NodeFilter, + "node-filter", + o.Filter.NodeFilter, + "comma separated list of nodes to snapshot", + ) + + saveCmd.Flags().StringVar( + &o.ToolsSnapshot.Format, + "format", + o.ToolsSnapshot.Format, + "output format: table, json", + ) + + saveCmd.Flags().DurationVar( + &o.ToolsSnapshot.Timeout, + "timeout", + o.ToolsSnapshot.Timeout, + "timeout per node for snapshot creation", + ) + + c.AddCommand(saveCmd) + return c, nil +} + +func snapshotSaveFn(cmd *cobra.Command, o *Options) error { + ctx := cmd.Context() + + // Initialize CLab + c, err := clabcore.NewContainerLab(o.ToClabOptions()...) + if err != nil { + return err + } + + // Create output directory + if err := os.MkdirAll(o.ToolsSnapshot.OutputDir, 0755); err != nil { + return fmt.Errorf("failed to create output directory: %w", err) + } + + // Get containers to snapshot (respects node-filter) + containers, err := c.ListNodesContainers(ctx) + if err != nil { + return err + } + + if len(containers) == 0 { + return fmt.Errorf("no containers found matching filters") + } + + // Create snapshot collection for tracking results + collection := NewSnapshotCollection() + + var wg sync.WaitGroup + + log.Infof("Creating snapshots for %d node(s)...", len(containers)) + + // Process each container concurrently + for idx := range containers { + wg.Add(1) + go func(container clabruntime.GenericContainer) { + defer wg.Done() + + nodeName := container.Labels[clabconstants.NodeName] + outputPath := filepath.Join(o.ToolsSnapshot.OutputDir, nodeName+".tar") + + result := createNodeSnapshot(ctx, container, outputPath, o.ToolsSnapshot.Timeout) + collection.Add(result) + + // Log result + if result.Error != nil { + log.Errorf("%s: %v", nodeName, result.Error) + } else if result.Status == "skipped" { + log.Infof("%s: skipping (%s)", nodeName, result.Reason) + } else { + log.Infof("%s: snapshot saved", + nodeName, + "path", result.SnapshotPath, + "size", formatBytes(result.SizeBytes), + "duration", result.Duration.Round(time.Second)) + } + }(containers[idx]) + } + + wg.Wait() + + // Print summary + log.Infof("Summary: %s", collection.Summary()) + + return nil +} + +// SnapshotResult represents the result of a snapshot operation for one node. +type SnapshotResult struct { + NodeName string + Status string // "success", "failed", "skipped" + SnapshotPath string + SizeBytes int64 + Duration time.Duration + Error error + Reason string +} + +// SnapshotCollection aggregates results from multiple node snapshot operations. +type SnapshotCollection struct { + results map[string]*SnapshotResult + m sync.RWMutex +} + +// NewSnapshotCollection creates a new snapshot collection. +func NewSnapshotCollection() *SnapshotCollection { + return &SnapshotCollection{ + results: make(map[string]*SnapshotResult), + } +} + +// Add adds a snapshot result to the collection. +func (sc *SnapshotCollection) Add(result *SnapshotResult) { + sc.m.Lock() + defer sc.m.Unlock() + sc.results[result.NodeName] = result +} + +// Summary returns a summary string of the snapshot operations. +func (sc *SnapshotCollection) Summary() string { + sc.m.RLock() + defer sc.m.RUnlock() + + var success, failed, skipped int + var totalSize int64 + + for _, r := range sc.results { + switch r.Status { + case "success": + success++ + totalSize += r.SizeBytes + case "failed": + failed++ + case "skipped": + skipped++ + } + } + + return fmt.Sprintf("%d succeeded, %d failed, %d skipped (%s total)", + success, failed, skipped, formatBytes(totalSize)) +} + +// createNodeSnapshot creates a snapshot for a single node. +func createNodeSnapshot(ctx context.Context, container clabruntime.GenericContainer, + outputPath string, timeout time.Duration) *SnapshotResult { + start := time.Now() + result := &SnapshotResult{ + NodeName: container.Labels[clabconstants.NodeName], + SnapshotPath: outputPath, + } + + // Check if this is a vrnetlab node + if !isVrnetlabNode(container) { + kind := container.Labels[clabconstants.NodeKind] + result.Status = "skipped" + result.Reason = fmt.Sprintf("not a vrnetlab node (kind=%s, image=%s)", kind, container.Image) + return result + } + + containerName := container.Names[0] + runtime := container.Runtime + + // 1. Trigger snapshot creation + log.Debugf("%s: triggering snapshot creation", result.NodeName) + execCmd := clabexec.NewExecCmdFromSlice([]string{"touch", "/snapshot-save"}) + if err := runtime.ExecNotWait(ctx, containerName, execCmd); err != nil { + result.Status = "failed" + result.Error = fmt.Errorf("failed to trigger snapshot: %w", err) + result.Duration = time.Since(start) + return result + } + + // 2. Wait for snapshot to complete + log.Debugf("%s: waiting for snapshot completion", result.NodeName) + if err := waitForSnapshotFile(ctx, runtime, containerName, timeout); err != nil { + result.Status = "failed" + result.Error = err + result.Duration = time.Since(start) + return result + } + + // 3. Copy snapshot from container to host + log.Debugf("%s: copying snapshot to host", result.NodeName) + if err := copySnapshotFromContainer(ctx, containerName, outputPath); err != nil { + result.Status = "failed" + result.Error = fmt.Errorf("failed to copy snapshot: %w", err) + result.Duration = time.Since(start) + return result + } + + // Get file size + if fi, err := os.Stat(outputPath); err == nil { + result.SizeBytes = fi.Size() + } + + result.Status = "success" + result.Duration = time.Since(start) + return result +} + +// waitForSnapshotFile waits for vrnetlab to complete snapshot creation by monitoring logs. +// Vrnetlab logs "Snapshot saved to /snapshot.tar" when the snapshot is complete. +func waitForSnapshotFile(ctx context.Context, runtime clabruntime.ContainerRuntime, + containerName string, timeout time.Duration) error { + deadline := time.Now().Add(timeout) + + // Get log stream from container + logReader, err := runtime.StreamLogs(ctx, containerName) + if err != nil { + return fmt.Errorf("failed to stream logs: %w", err) + } + defer logReader.Close() + + // Create a buffered reader to read logs line by line + logScanner := make(chan string, 100) + errChan := make(chan error, 1) + + // Start goroutine to read logs + go func() { + defer close(logScanner) + buf := make([]byte, 4096) + var partial string + + for { + n, err := logReader.Read(buf) + if n > 0 { + // Combine with any partial line from previous read + text := partial + string(buf[:n]) + lines := strings.Split(text, "\n") + + // Last element might be incomplete + partial = lines[len(lines)-1] + + // Send complete lines + for i := 0; i < len(lines)-1; i++ { + select { + case logScanner <- lines[i]: + case <-ctx.Done(): + return + } + } + } + if err != nil { + if err.Error() != "EOF" { + errChan <- err + } + return + } + } + }() + + // Wait for snapshot completion message + for { + select { + case <-ctx.Done(): + return ctx.Err() + + case err := <-errChan: + return fmt.Errorf("error reading logs: %w", err) + + case line, ok := <-logScanner: + if !ok { + return fmt.Errorf("log stream closed before snapshot completed") + } + + // Check for failure message first + if strings.Contains(line, "Snapshot save failed:") { + return fmt.Errorf("snapshot save failed: %s", line) + } + + // Check for completion message (vrnetlab now saves to /snapshot-output.tar) + if strings.Contains(line, "Snapshot saved to /snapshot-output.tar") { + log.Debugf("%s: snapshot creation complete", containerName) + return nil + } + + // Also log any errors from vrnetlab + if strings.Contains(line, "ERROR") || strings.Contains(line, "Error") { + log.Debugf("%s: %s", containerName, line) + } + + case <-time.After(timeout): + return fmt.Errorf("timeout waiting for snapshot after %v", timeout) + } + + if time.Now().After(deadline) { + return fmt.Errorf("timeout waiting for snapshot after %v", timeout) + } + } +} + +// copySnapshotFromContainer copies the snapshot file from container to host. +func copySnapshotFromContainer(ctx context.Context, containerName, outputPath string) error { + // Use docker cp command to copy snapshot from container + // vrnetlab saves to /snapshot-output.tar when triggered by /snapshot-save + cmd := exec.CommandContext(ctx, "docker", "cp", + containerName+":/snapshot-output.tar", + outputPath) + + if output, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("docker cp failed: %w, output: %s", err, string(output)) + } + + return nil +} + +// isVrnetlabKind checks if a node is a vrnetlab-based node. +// It checks if the node's image starts with "vrnetlab/". +func isVrnetlabNode(container clabruntime.GenericContainer) bool { + // Check if image starts with vrnetlab/ + return strings.HasPrefix(container.Image, "vrnetlab/") +} + +// formatBytes formats bytes into human-readable format. +func formatBytes(bytes int64) string { + const unit = 1024 + if bytes < unit { + return fmt.Sprintf("%d B", bytes) + } + div, exp := int64(unit), 0 + for n := bytes / unit; n >= unit; n /= unit { + div *= unit + exp++ + } + return fmt.Sprintf("%.1f %cB", float64(bytes)/float64(div), "KMGTPE"[exp]) +} diff --git a/core/deploy.go b/core/deploy.go index 5733614d5e..eef1227064 100644 --- a/core/deploy.go +++ b/core/deploy.go @@ -2,7 +2,10 @@ package core import ( "context" + "fmt" "os" + "path/filepath" + "strings" "sync" "time" @@ -130,6 +133,11 @@ func (c *CLab) Deploy( //nolint: funlen n.Config().ExtraHosts = extraHosts } + // Apply snapshot restore configuration to nodes + if err := c.configureSnapshotRestore(options); err != nil { + return nil, err + } + nodesWg, execCollection, err := c.createNodes(ctx, options.maxWorkers, options.skipPostDeploy) if err != nil { return nil, err @@ -293,3 +301,89 @@ func (c *CLab) createNodes( return NodesWg, execCollection, nil } + +// configureSnapshotRestore configures nodes for snapshot restoration. +// It resolves snapshot files for each node and adds the necessary volume mounts +// and environment variables to restore from snapshots. +func (c *CLab) configureSnapshotRestore(options *DeployOptions) error { + // Build restore map from per-node specifications + restoreMap := make(map[string]string) + for _, mapping := range options.restoreNodeSnapshots { + parts := strings.SplitN(mapping, "=", 2) + if len(parts) != 2 { + return fmt.Errorf("invalid restore format: %s (expected: node=path)", mapping) + } + restoreMap[parts[0]] = parts[1] + } + + // Track statistics for logging + var restoredCount, freshCount int + + // Configure each node + for _, node := range c.Nodes { + nodeName := node.Config().ShortName + + // Resolve snapshot path for this node + snapshotPath, shouldRestore := resolveNodeSnapshot(nodeName, restoreMap, options.restoreAll) + + if shouldRestore { + // Validate snapshot file exists + if _, err := os.Stat(snapshotPath); os.IsNotExist(err) { + return fmt.Errorf("snapshot file not found for node %s: %s", nodeName, snapshotPath) + } + + // Get absolute path for bind mount + absPath, err := filepath.Abs(snapshotPath) + if err != nil { + return fmt.Errorf("failed to get absolute path for %s: %w", snapshotPath, err) + } + + // Add volume mount for snapshot (read-only) + node.Config().Binds = append(node.Config().Binds, + fmt.Sprintf("%s:/snapshot.tar:ro", absPath)) + + // Add restore environment variable + if node.Config().Env == nil { + node.Config().Env = make(map[string]string) + } + node.Config().Env["RESTORE_SNAPSHOT"] = "1" + + log.Infof("Node %s will restore from: %s", nodeName, snapshotPath) + restoredCount++ + } else { + freshCount++ + } + } + + // Log deployment mode summary if any restores are configured + if restoredCount > 0 { + log.Infof("Deploying %d nodes from snapshots, %d nodes fresh", restoredCount, freshCount) + } + + return nil +} + +// resolveNodeSnapshot determines the snapshot file path for a given node. +// Priority: per-node override > restore-all directory > none. +func resolveNodeSnapshot(nodeName string, restoreMap map[string]string, restoreAll string) (string, bool) { + // 1. Check per-node overrides first (highest priority) + if snapshotPath, ok := restoreMap[nodeName]; ok { + return snapshotPath, true + } + + // 2. Check restore-all directory + if restoreAll != "" { + // Look for {nodename}.tar in directory + snapshotPath := filepath.Join(restoreAll, nodeName+".tar") + if _, err := os.Stat(snapshotPath); err == nil { + return snapshotPath, true + } + + // Not found - this is OK, just deploy normally + log.Debugf("No snapshot found for node %s in %s, deploying normally", nodeName, restoreAll) + return "", false + } + + // 3. No restore specified + return "", false +} diff --git a/core/options_deploy.go b/core/options_deploy.go index 8b01a232f5..ef835256a7 100644 --- a/core/options_deploy.go +++ b/core/options_deploy.go @@ -7,12 +7,14 @@ import ( // DeployOptions represents the options for deploying a lab. type DeployOptions struct { - reconfigure bool // reconfigure indicates whether to reconfigure the lab. - skipPostDeploy bool // skipPostDeploy indicates whether to skip post-deployment steps. - graph bool // graph indicates whether to generate a graph of the lab. - maxWorkers uint // maxWorkers is the maximum number of workers for node creation. - exportTemplate string // exportTemplate is the path to the export template. - skipLabDirFileACLs bool // skip setting the extended File ACL entries on the lab directory. + reconfigure bool // reconfigure indicates whether to reconfigure the lab. + skipPostDeploy bool // skipPostDeploy indicates whether to skip post-deployment steps. + graph bool // graph indicates whether to generate a graph of the lab. + maxWorkers uint // maxWorkers is the maximum number of workers for node creation. + exportTemplate string // exportTemplate is the path to the export template. + skipLabDirFileACLs bool // skip setting the extended File ACL entries on the lab directory. + restoreAll string // restoreAll specifies a directory to scan for snapshot files. + restoreNodeSnapshots []string // restoreNodeSnapshots maps node names to specific snapshot file paths. } // NewDeployOptions creates a new DeployOptions instance with the specified maxWorkers value. @@ -89,6 +91,28 @@ func (d *DeployOptions) ExportTemplate() string { return d.exportTemplate } +// SetRestoreAll sets the restoreAll option and returns the updated DeployOptions instance. +func (d *DeployOptions) SetRestoreAll(path string) *DeployOptions { + d.restoreAll = path + return d +} + +// RestoreAll returns the restoreAll option value. +func (d *DeployOptions) RestoreAll() string { + return d.restoreAll +} + +// SetRestoreNodeSnapshots sets the restoreNodeSnapshots option and returns the updated DeployOptions instance. +func (d *DeployOptions) SetRestoreNodeSnapshots(snapshots []string) *DeployOptions { + d.restoreNodeSnapshots = snapshots + return d +} + +// RestoreNodeSnapshots returns the restoreNodeSnapshots option value. +func (d *DeployOptions) RestoreNodeSnapshots() []string { + return d.restoreNodeSnapshots +} + // initWorkerCount calculates the number of workers used for node creation. // If maxWorkers is provided, it takes precedence. // If maxWorkers is not set, the number of workers is limited by the number of available CPUs diff --git a/docs/cmd/deploy.md b/docs/cmd/deploy.md index 4a6c14dc81..401318a9b0 100644 --- a/docs/cmd/deploy.md +++ b/docs/cmd/deploy.md @@ -166,6 +166,58 @@ Example: containerlab deploy -t mylab.clab.yml --owner alice ``` +#### restore-all + +The local `--restore-all` flag enables restoring vrnetlab-based nodes from previously saved snapshots. When specified, containerlab will look for snapshot files named `{nodename}.tar` in the provided directory and automatically restore nodes that have matching snapshots. + +Nodes without snapshots in the directory will deploy normally (fresh deployment). + +**Default directory**: `./snapshots` (if flag is used without a value) + +```bash +# Restore all nodes that have snapshots in ./snapshots directory +containerlab deploy -t mylab.clab.yml --restore-all + +# Restore from custom directory +containerlab deploy -t mylab.clab.yml --restore-all /backups/lab1 +``` + +**Note**: Only vrnetlab-based nodes support snapshot restore. The snapshot feature requires vrnetlab images with snapshot support. + +#### restore + +The local `--restore` flag allows per-node snapshot restoration by explicitly specifying the snapshot file path for individual nodes. This flag can be specified multiple times to restore different nodes from different snapshot files. + +**Format**: `--restore node=path/to/snapshot.tar` + +```bash +# Restore only r1 from a specific snapshot +containerlab deploy -t mylab.clab.yml --restore r1=./snapshots/r1.tar + +# Restore multiple nodes with specific snapshots +containerlab deploy -t mylab.clab.yml \ + --restore r1=./snapshots/r1.tar \ + --restore r2=./snapshots/r2.tar +``` + +**Priority**: Per-node `--restore` specifications override `--restore-all` for the specified nodes. + +**Combined usage**: + +```bash +# Restore all from ./snapshots, but override r3 with a different snapshot +containerlab deploy -t mylab.clab.yml \ + --restore-all ./snapshots \ + --restore r3=./backups/r3-old.tar +``` + +In this example: +- Nodes with snapshots in `./snapshots/` will restore from there +- Node `r3` will restore from `./backups/r3-old.tar` (override) +- Nodes without snapshots will deploy fresh + +See [tools snapshot save](tools/snapshot/save.md) for information on creating snapshots. + ### Environment variables #### `CLAB_RUNTIME` diff --git a/docs/cmd/tools/snapshot/save.md b/docs/cmd/tools/snapshot/save.md new file mode 100644 index 0000000000..35907b7a76 --- /dev/null +++ b/docs/cmd/tools/snapshot/save.md @@ -0,0 +1,75 @@ +# tools snapshot save + +## Description + +The `tools snapshot save` command creates snapshots of running vrnetlab-based VMs and saves them to disk. Snapshots capture the complete VM state including configuration, running processes, and memory, allowing for fast restoration later. + +Each node's snapshot is saved as `{output-dir}/{nodename}.tar`. + +Only vrnetlab-based nodes support snapshots. Non-vrnetlab nodes are automatically skipped. + +## Usage + +`containerlab tools snapshot save [flags]` + +## Flags + +### output-dir + +Specify the directory where snapshot files will be saved. + +Default value is `./snapshots`. + +```bash +containerlab tools snapshot save --output-dir /backups/lab1 +``` + +Creates snapshot files as `/backups/lab1/{nodename}.tar` + +### node-filter + +Specify a comma-separated list of nodes to snapshot. Only the specified nodes will be processed. + +```bash +containerlab tools snapshot save --node-filter r1,r2,r3 +``` + +### timeout + +Set the maximum time to wait for each node's snapshot creation. + +Default value is `5m` (5 minutes). + +```bash +containerlab tools snapshot save --timeout 10m +``` + +### format + +Output format for the results summary. Possible values: `table`, `json`. + +Default value is `table`. + +## Examples + +### Save all vrnetlab nodes + +```bash +containerlab tools snapshot save -t mylab.clab.yml +``` + +Creates `./snapshots/r1.tar`, `./snapshots/r2.tar`, etc. + +### Save specific nodes to custom directory + +```bash +containerlab tools snapshot save -t mylab.clab.yml \ + --node-filter r1,r2 \ + --output-dir /backups/2024-01-15 +``` + +### Save with extended timeout + +```bash +containerlab tools snapshot save -t mylab.clab.yml --timeout 10m +```