feat(deployment-policy): add batch state reset with auto-reset, CLI, and config

Add deployment policy batch state reset to ensure rollouts start fresh
instead of continuing with scaled-up batch sizes. Includes:

- Auto-reset on rollout completion and spec version change
- Manual CLI command: `deployment-policy reset` (alias: `dp reset`)
- `resetBatchStateOnCompletion` field on DeploymentPolicy and Skyhook
- Config precedence: Skyhook override > DeploymentPolicy > default
- `--skip-batch-reset` flag on existing `reset` command
- E2E tests for auto-reset, config precedence, and CLI reset
- Documentation for deployment policy and CLI

Author: lockwobr

chart/templates/cleanup-skyhooks-job.yaml

@@ -19,6 +19,27 @@ spec:
       tolerations:
         {{- toYaml . | nindent 6 }}
       {{- end }}
+      {{- with .Values.controllerManager.selectors }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- if .Values.controllerManager.nodeAffinity.matchExpressions }}
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              {{- range .Values.controllerManager.nodeAffinity.matchExpressions }}
+              - key: {{ .key }}
+                operator: {{ .operator }}
+                {{- if .values }}
+                values:
+                {{- range .values }}
+                - {{ . }}
+                {{- end }}
+                {{- end }}
+              {{- end }}
+      {{- end }}
       securityContext:
         runAsNonRoot: true
         runAsUser: 10001
@@ -59,13 +80,23 @@ spec:
         - mountPath: /var/run/secrets/kubernetes.io/serviceaccount
           name: kube-api-access
           readOnly: true
+        # Guard against limitRange being disabled (set to null/false) with fallback defaults
         resources:
+        {{- if .Values.limitRange }}
           limits:
             cpu: {{ .Values.limitRange.default.cpu }}
             memory: {{ .Values.limitRange.default.memory }}
           requests:
             cpu: {{ .Values.limitRange.defaultRequest.cpu }}
             memory: {{ .Values.limitRange.defaultRequest.memory }}
+        {{- else }}
+          limits:
+            cpu: 500m
+            memory: 512Mi
+          requests:
+            cpu: 250m
+            memory: 256Mi
+        {{- end }}
         command:
         - /bin/sh
         - -c

chart/templates/cleanup-webhook-job.yaml

@@ -14,6 +14,31 @@ spec:
       restartPolicy: Never
       automountServiceAccountToken: false
       serviceAccountName: {{ include "chart.fullname" . }}-controller-manager
+      {{- with .Values.controllerManager.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 6 }}
+      {{- end }}
+      {{- with .Values.controllerManager.selectors }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- if .Values.controllerManager.nodeAffinity.matchExpressions }}
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              {{- range .Values.controllerManager.nodeAffinity.matchExpressions }}
+              - key: {{ .key }}
+                operator: {{ .operator }}
+                {{- if .values }}
+                values:
+                {{- range .values }}
+                - {{ . }}
+                {{- end }}
+                {{- end }}
+              {{- end }}
+      {{- end }}
       securityContext:
         runAsNonRoot: true
         runAsUser: 10001
@@ -54,13 +79,23 @@ spec:
         - mountPath: /var/run/secrets/kubernetes.io/serviceaccount
           name: kube-api-access
           readOnly: true
+        # Guard against limitRange being disabled (set to null/false) with fallback defaults
         resources:
+        {{- if .Values.limitRange }}
           limits:
             cpu: {{ .Values.limitRange.default.cpu }}
             memory: {{ .Values.limitRange.default.memory }}
           requests:
             cpu: {{ .Values.limitRange.defaultRequest.cpu }}
             memory: {{ .Values.limitRange.defaultRequest.memory }}
+        {{- else }}
+          limits:
+            cpu: 500m
+            memory: 512Mi
+          requests:
+            cpu: 250m
+            memory: 256Mi
+        {{- end }}
         command:
         - /bin/sh
         - -c

chart/templates/deploymentpolicy-crd.yaml

@@ -267,6 +267,11 @@ spec:
                 required:
                 - strategy
                 type: object
+              resetBatchStateOnCompletion:
+                default: false
+                description: ResetBatchStateOnCompletion controls whether batch state
+                  is reset when rollout completes or spec changes
+                type: boolean
             required:
             - default
             type: object

chart/templates/skyhook-crd.yaml

@@ -115,6 +115,16 @@ spec:
                 description: DeploymentPolicy is the name of a DeploymentPolicy for
                   rollout settings
                 type: string
+              deploymentPolicyOptions:
+                description: DeploymentPolicyOptions allows per-Skyhook overrides
+                  of DeploymentPolicy settings
+                properties:
+                  resetBatchStateOnCompletion:
+                    default: false
+                    description: ResetBatchStateOnCompletion overrides the DeploymentPolicy
+                      setting for this Skyhook
+                    type: boolean
+                type: object
               interruptionBudget:
                 description: InterruptionBudget configures how many nodes that match
                   node selectors that allowed to be interrupted at once.

docs/cli.md

@@ -25,6 +25,7 @@ The CLI requires **operator version v0.8.0 or later** for full functionality of
 | `package rerun` | ✅ Full | ✅ Full |
 | `package logs` | ✅ Full | ✅ Full |
 | `reset` | ✅ Full | ✅ Full |
+| `deployment-policy reset` | ❌ Not supported | ✅ Full |
 | `pause` | ❌ Not supported | ✅ Full |
 | `resume` | ❌ Not supported | ✅ Full |
 | `disable` | ❌ Not supported | ✅ Full |
@@ -126,6 +127,62 @@ kubectl skyhook disable my-skyhook --confirm
 kubectl skyhook enable my-skyhook
 ```
 
+### Reset Command
+
+Reset all package state for a Skyhook, causing re-execution from the beginning.
+
+```bash
+# Reset all nodes for a Skyhook (also resets batch state by default)
+kubectl skyhook reset gpu-init --confirm
+
+# Preview changes without applying (dry-run)
+kubectl skyhook reset gpu-init --dry-run
+
+# Reset nodes only, preserve deployment policy batch state
+kubectl skyhook reset gpu-init --skip-batch-reset --confirm
+```
+
+| Flag | Description |
+|------|-------------|
+| `--confirm, -y` | Skip confirmation prompt |
+| `--skip-batch-reset` | Skip resetting deployment policy batch state |
+
+> **Note:** By default, `reset` also resets the deployment policy batch state so the next rollout starts from batch 1. Use `--skip-batch-reset` to preserve the existing batch state.
+
+### Deployment Policy Commands
+
+Manage deployment policy batch state.
+
+> **Note:** Requires operator v0.8.0+.
+
+```bash
+# Reset batch state for a Skyhook (starts rollout from batch 1)
+kubectl skyhook deployment-policy reset gpu-init --confirm
+
+# Preview what would be reset (dry-run)
+kubectl skyhook deployment-policy reset gpu-init --dry-run
+
+# Using the short alias
+kubectl skyhook dp reset gpu-init --confirm
+```
+
+The `deployment-policy reset` command resets the batch processing state for all compartments in the specified Skyhook, including:
+- Current batch number (reset to 1)
+- Consecutive failure count
+- Completed and failed node counts
+- Stop flag
+
+| Flag | Description |
+|------|-------------|
+| `--confirm, -y` | Skip confirmation prompt |
+
+**When to use**:
+- After a rollout completes and you want to start a new rollout fresh
+- When batch processing is stuck and needs to be reset
+- Before re-running a rollout with the same deployment policy
+
+See [Deployment Policy documentation](deployment_policy.md) for details on auto-reset configuration.
+
 ### Node Commands
 
 Manage Skyhook nodes across the cluster.
@@ -208,10 +265,12 @@ kubectl skyhook --help
 # Command group help
 kubectl skyhook node --help
 kubectl skyhook package --help
+kubectl skyhook deployment-policy --help
 
 # Specific command help
 kubectl skyhook node reset --help
 kubectl skyhook package rerun --help
+kubectl skyhook deployment-policy reset --help
 ```
 
 ## Common Usage Patterns
@@ -252,6 +311,18 @@ kubectl skyhook node status
 kubectl skyhook node status --skyhook my-skyhook -o json
 ```
 
+### Resetting a Rollout
+```bash
+# 1. Full reset: nodes + batch state (starts everything fresh)
+kubectl skyhook reset my-skyhook --confirm
+
+# 2. Or reset only batch state (keep node state, restart batch progression)
+kubectl skyhook deployment-policy reset my-skyhook --confirm
+
+# 3. Or reset only nodes (keep batch progression)
+kubectl skyhook reset my-skyhook --skip-batch-reset --confirm
+```
+
 ### Emergency Stop
 
 > **Note:** Requires operator v0.8.0+. For older operators, use `kubectl edit skyhook my-skyhook` and set `spec.pause: true`.
@@ -289,7 +360,11 @@ kubectl skyhook node list --skyhook my-skyhook -o yaml
 operator/cmd/cli/app/           # CLI commands
 ├── cli.go                      # Root command (NewSkyhookCommand)
 ├── version.go                  # Version command
+├── reset.go                    # Reset command (nodes + batch state)
 ├── lifecycle.go                # Pause, resume, disable, enable commands
+├── deploymentpolicy/           # Deployment policy subcommands
+│   ├── deploymentpolicy.go     # Parent command
+│   └── deploymentpolicy_reset.go  # Batch state reset
 ├── node/                       # Node subcommands
 │   ├── node.go                 # Parent command
 │   ├── node_list.go
@@ -314,10 +389,13 @@ main()
   └── cli.Execute()
       └── NewSkyhookCommand(ctx)
           ├── NewVersionCmd(ctx)
+          ├── NewResetCmd(ctx)
           ├── NewPauseCmd(ctx)
           ├── NewResumeCmd(ctx)
           ├── NewDisableCmd(ctx)
           ├── NewEnableCmd(ctx)
+          ├── deploymentpolicy.NewDeploymentPolicyCmd(ctx)
+          │   └── NewResetCmd(ctx)
           ├── node.NewNodeCmd(ctx)
           │   ├── NewListCmd(ctx)
           │   ├── NewStatusCmd(ctx)

docs/deployment_policy.md

@@ -23,6 +23,8 @@ kind: DeploymentPolicy
 metadata:
   name: my-policy
 spec:
+  # Reset batch state automatically when rollout completes or spec version changes
+  resetBatchStateOnCompletion: true  # default: true
   # Default applies to nodes that don't match any compartment
   default:
     budget:
@@ -240,6 +242,94 @@ compartments:
 
 ---
 
+## Batch State Reset
+
+When using progressive rollout strategies (linear, exponential), the operator tracks batch processing state per compartment — current batch number, consecutive failures, completed/failed node counts, etc. This state persists across reconciliations so the rollout can scale up progressively.
+
+However, when a rollout **completes** or a **spec version changes**, you typically want the next rollout to start fresh from batch 1 rather than continuing with scaled-up batch sizes. Batch state reset handles this automatically.
+
+### Auto-Reset Triggers
+
+Batch state is automatically reset when **either** of these events occurs (if configured):
+
+1. **Rollout completion** — When a Skyhook's status transitions to `Complete`
+2. **Spec version change** — When a package version changes in the Skyhook spec
+
+After reset, the next reconciliation starts from batch 1 with all counters cleared.
+
+### Configuration
+
+Auto-reset is controlled by two fields with a precedence hierarchy:
+
+| Field | Location | Description |
+|-------|----------|-------------|
+| `spec.resetBatchStateOnCompletion` | DeploymentPolicy | Default setting for all Skyhooks using this policy |
+| `spec.deploymentPolicyOptions.resetBatchStateOnCompletion` | Skyhook | Per-Skyhook override (takes precedence) |
+
+**Precedence order** (highest to lowest):
+1. Skyhook's `deploymentPolicyOptions.resetBatchStateOnCompletion`
+2. DeploymentPolicy's `resetBatchStateOnCompletion`
+3. Default: `true` (safe by default for new resources)
+
+### Examples
+
+**Enable auto-reset (default behavior for new policies)**:
+```yaml
+apiVersion: skyhook.nvidia.com/v1alpha1
+kind: DeploymentPolicy
+metadata:
+  name: my-policy
+spec:
+  resetBatchStateOnCompletion: true  # Enabled by default
+  default:
+    budget:
+      percent: 25
+```
+
+**Disable auto-reset for a specific Skyhook** (override the policy):
+```yaml
+apiVersion: skyhook.nvidia.com/v1alpha1
+kind: Skyhook
+metadata:
+  name: my-skyhook
+spec:
+  deploymentPolicy: my-policy
+  deploymentPolicyOptions:
+    resetBatchStateOnCompletion: false  # Override: keep batch state across rollouts
+```
+
+**Disable auto-reset at the policy level**:
+```yaml
+apiVersion: skyhook.nvidia.com/v1alpha1
+kind: DeploymentPolicy
+metadata:
+  name: preserve-state-policy
+spec:
+  resetBatchStateOnCompletion: false  # All Skyhooks using this policy keep batch state
+```
+
+### Manual Reset
+
+You can also reset batch state manually using the CLI:
+
+```bash
+# Reset batch state for a specific Skyhook
+kubectl skyhook deployment-policy reset my-skyhook --confirm
+
+# Preview what would be reset (dry-run)
+kubectl skyhook deployment-policy reset my-skyhook --dry-run
+
+# The 'reset' command also resets batch state by default
+kubectl skyhook reset my-skyhook --confirm
+
+# To reset nodes only without resetting batch state
+kubectl skyhook reset my-skyhook --skip-batch-reset --confirm
+```
+
+See [CLI documentation](cli.md) for full command details.
+
+---
+
 ## Using with Skyhooks
 
 Reference a policy by name:
@@ -251,6 +341,8 @@ metadata:
   name: my-skyhook
 spec:
   deploymentPolicy: my-policy  # References DeploymentPolicy
+  deploymentPolicyOptions:     # Optional per-Skyhook overrides
+    resetBatchStateOnCompletion: true
   nodeSelectors:
     matchLabels:
       workload: gpu
@@ -262,6 +354,7 @@ spec:
 - DeploymentPolicy is **cluster-scoped** (not namespaced)
 - Each node is assigned to a compartment based on selectors
 - Nodes not matching any compartment use the `default` settings
+- `deploymentPolicyOptions` allows per-Skyhook overrides of policy settings
 
 ---