Merge branch 'kubernetes:master' into kep-hpa-tolerance

Author: jm-franc

api/approval.go

@@ -41,7 +41,8 @@ type PRRApproval struct {
 	Beta       *PRRMilestone `json:"beta" yaml:"beta,omitempty"`
 	Stable     *PRRMilestone `json:"stable" yaml:"stable,omitempty"`
 	Deprecated *PRRMilestone `json:"deprecated" yaml:"deprecated,omitempty"`
-	Removed    *PRRMilestone `json:"removed", yaml:"removed,omitempty"`
+	Removed    *PRRMilestone `json:"removed" yaml:"removed,omitempty"`
+	Disabled   *PRRMilestone `json:"disabled" yaml:"disabled,omitempty"`
 
 	// TODO(api): Move to separate struct for handling document parsing
 	Error error `json:"-" yaml:"-"`
@@ -61,7 +62,10 @@ func (prr *PRRApproval) ApproverForStage(stage Stage) (string, error) {
 		return "", err
 	}
 
-	if prr.Alpha == nil && prr.Beta == nil && prr.Stable == nil {
+	// KEPs are usually of 2 types:
+	// 1. Features that go through Alpha->Beta->Stable
+	// 2. Removals that go through Deprecated->Disabled->Removed
+	if prr.Alpha == nil && prr.Beta == nil && prr.Stable == nil && prr.Deprecated == nil && prr.Disabled == nil && prr.Removed == nil {
 		return "", ErrPRRMilestonesAllEmpty
 	}
 
@@ -70,20 +74,37 @@ func (prr *PRRApproval) ApproverForStage(stage Stage) (string, error) {
 		if prr.Alpha == nil {
 			return "", ErrPRRMilestoneIsNil
 		}
-
 		return prr.Alpha.Approver, nil
+
 	case BetaStage:
 		if prr.Beta == nil {
 			return "", ErrPRRMilestoneIsNil
 		}
-
 		return prr.Beta.Approver, nil
+
 	case StableStage:
 		if prr.Stable == nil {
 			return "", ErrPRRMilestoneIsNil
 		}
-
 		return prr.Stable.Approver, nil
+
+	case Deprecated:
+		if prr.Deprecated == nil {
+			return "", ErrPRRMilestoneIsNil
+		}
+		return prr.Deprecated.Approver, nil
+
+	case Disabled:
+		if prr.Disabled == nil {
+			return "", ErrPRRMilestoneIsNil
+		}
+		return prr.Disabled.Approver, nil
+
+	case Removed:
+		if prr.Removed == nil {
+			return "", ErrPRRMilestoneIsNil
+		}
+		return prr.Removed.Approver, nil
 	}
 
 	return "", ErrPRRApproverUnknown

api/approval_test.go

@@ -40,6 +40,18 @@ var (
 		Stable: prrMilestoneWithApprover,
 	}
 
+	validDeprecatedPRR = &api.PRRApproval{
+		Deprecated: prrMilestoneWithApprover,
+	}
+
+	validDisabledPRR = &api.PRRApproval{
+		Disabled: prrMilestoneWithApprover,
+	}
+
+	validRemovedPRR = &api.PRRApproval{
+		Removed: prrMilestoneWithApprover,
+	}
+
 	invalidPRR = &api.PRRApproval{}
 )
 
@@ -85,6 +97,27 @@ func TestPRRApproval_ApproverForStage(t *testing.T) {
 			wantApprover: "@wojtek-t",
 			wantErr:      false,
 		},
+		{
+			name:         "valid: deprecated",
+			stage:        "deprecated",
+			prr:          validDeprecatedPRR,
+			wantApprover: "@wojtek-t",
+			wantErr:      false,
+		},
+		{
+			name:         "valid: disabled",
+			stage:        "disabled",
+			prr:          validDisabledPRR,
+			wantApprover: "@wojtek-t",
+			wantErr:      false,
+		},
+		{
+			name:         "valid: removed",
+			stage:        "removed",
+			prr:          validRemovedPRR,
+			wantApprover: "@wojtek-t",
+			wantErr:      false,
+		},
 	}
 
 	for _, tc := range testcases {

api/proposal.go

@@ -36,12 +36,18 @@ const (
 	AlphaStage  Stage = "alpha"
 	BetaStage   Stage = "beta"
 	StableStage Stage = "stable"
+	Deprecated  Stage = "deprecated"
+	Disabled    Stage = "disabled"
+	Removed     Stage = "removed"
 )
 
 var ValidStages = []Stage{
 	AlphaStage,
 	BetaStage,
 	StableStage,
+	Deprecated,
+	Disabled,
+	Removed,
 }
 
 func (s Stage) IsValid() error {
@@ -138,6 +144,7 @@ type Milestone struct {
 	Stable     string `json:"stable" yaml:"stable"`
 	Deprecated string `json:"deprecated" yaml:"deprecated,omitempty"`
 	Removed    string `json:"removed" yaml:"removed,omitempty"`
+	Disabled   string `json:"disabled" yaml:"disabled,omitempty"`
 }
 
 type FeatureGate struct {

keps/NNNN-kep-template/kep.yaml

@@ -22,6 +22,8 @@ replaces:
   - "/keps/sig-ccc/3456-replaced-kep"
 
 # The target maturity stage in the current dev cycle for this KEP.
+# If the purpose of this KEP is to deprecate a user-visible feature
+# and a Deprecated feature gates are added, they should be deprecated|disabled|removed.
 stage: alpha|beta|stable
 
 # The most recent milestone for which work toward delivery of this KEP has been

keps/prod-readiness/sig-node/3386.yaml

@@ -1,5 +1,3 @@
 kep-number: 3386
 alpha:
   approver: "@deads2k"
-beta:
-  approver: "@deads2k"

keps/sig-apps/1847-autoremove-statefulset-pvcs/kep.yaml

@@ -7,7 +7,7 @@ authors:
 owning-sig: sig-apps
 participating-sigs:
   - sig-storage
-status: implementable
+status: implemented
 creation-date: 2020-06-04
 reviewers:
   - "@kow3ns"  

keps/sig-apps/3973-consider-terminating-pods-deployment/README.md

@@ -19,6 +19,7 @@
 - [Design Details](#design-details)
   - [Deployment Behavior Changes](#deployment-behavior-changes)
   - [ReplicaSet Status and Deployment Status Changes](#replicaset-status-and-deployment-status-changes)
+  - [Deployment Completion and Progress Changes](#deployment-completion-and-progress-changes)
   - [Deployment Scaling Changes and a New Annotation for ReplicaSets](#deployment-scaling-changes-and-a-new-annotation-for-replicasets)
   - [kubectl Changes](#kubectl-changes)
   - [API](#api)
@@ -279,6 +280,20 @@ To satisfy the requirement for tracking terminating pods, and for implementation
 we propose a new field `.status.terminatingReplicas` to the ReplicaSet's and Deployment's
 status.
 
+### Deployment Completion and Progress Changes
+
+Currently, when the latest ReplicaSet is fully saturated and all of its pods become available, the
+Deployment is declared complete. However, there may still be old terminating pods. These pods can
+still be ready and hold/accept connections, meaning that the transition to the latest revision is
+not fully complete.
+
+To avoid unexpected behavior, we should not declare the deployment complete until all of its
+terminating replicas have been fully terminated. We will therefore delay setting a `NewRSAvailable`
+reason to the `DeploymentProgressing` condition, when `TerminationComplete` policy is used.
+
+We will also update the `LastUpdateTime` of the `DeploymentProgressing` condition when the number of
+terminating pods decreases to reset the progress deadline.
+
 ### Deployment Scaling Changes and a New Annotation for ReplicaSets
 
 Currently, scaling is done proportionally over all ReplicaSets to mitigate the risk of losing
@@ -383,14 +398,17 @@ See [kubectl Skew](#kubectl-skew) for more details.
 type DeploymentPodReplacementPolicy string
 const (
 // TerminationStarted policy creates replacement Pods when the old Pods start
-// terminating (has a .metadata.deletionTimestamp). The total number of
-// Deployment Pods can be greater than specified by the Deployment's
+// terminating (have a non-null .metadata.deletionTimestamp). The total number
+// of Deployment Pods can be greater than specified by the Deployment's
 // .spec.replicas and the DeploymentStrategy.
 TerminationStarted DeploymentPodReplacementPolicy = "TerminationStarted"
 // TerminationComplete policy creates replacement Pods only when the old Pods
 // are fully terminated (reach Succeeded or Failed phase). The old Pods are
 // subsequently removed. The total number of the Deployment Pods is
 // limited by the Deployment's .spec.replicas and the DeploymentStrategy.
+//
+// This policy will also delay declaring the deployment as complete until all
+// of its terminating replicas have been fully terminated.
 TerminationComplete DeploymentPodReplacementPolicy = "TerminationComplete"
 )
 ```
@@ -401,13 +419,15 @@ type DeploymentSpec struct {
     // podReplacementPolicy specifies when to create replacement Pods. 
 	// Possible values are:
     // - TerminationStarted policy creates replacement Pods when the old Pods start
-	//   terminating (has a .metadata.deletionTimestamp). The total number of
-	//   Deployment Pods can be greater than specified by the Deployment's
+	//   terminating (have a non-null .metadata.deletionTimestamp). The total number
+	//   of Deployment Pods can be greater than specified by the Deployment's
 	//   .spec.replicas and the DeploymentStrategy.
     // - TerminationComplete policy creates replacement Pods only when the old Pods
 	//   are fully terminated (reach Succeeded or Failed phase). The old Pods are
 	//   subsequently removed. The total number of the Deployment Pods is
 	//   limited by the Deployment's .spec.replicas and the DeploymentStrategy.
+	//   This policy will also delay declaring the deployment as complete until all
+	//   of its terminating replicas have been fully terminated.
     //
     // The default behavior when the policy is not specified depends on the DeploymentStrategy:
 	// - Recreate strategy uses TerminationComplete behavior when recreating the deployment,
@@ -442,11 +462,11 @@ type ReplicaSetStatus struct {
     // +optional
     AvailableReplicas int32 `json:"availableReplicas,omitempty" protobuf:"varint,5,opt,name=availableReplicas"`
 
-    // The number of terminating replicas (have .metadata.deletionTimestamp) for this replica set.
+    // The number of terminating pods (have a non-null .metadata.deletionTimestamp) for this replica set. 
     //
     // This is an alpha field. Enable DeploymentPodReplacementPolicy to be able to use this field.
     // +optional
-    TerminatingReplicas int32 `json:"terminatingReplicas,omitempty" protobuf:"varint,7,opt,name=terminatingReplicas"`
+    TerminatingReplicas *int32 `json:"terminatingReplicas,omitempty" protobuf:"varint,7,opt,name=terminatingReplicas"`
     ...
 }
 ```
@@ -476,11 +496,11 @@ type DeploymentStatus struct {
     // +optional
     UnavailableReplicas int32 `json:"unavailableReplicas,omitempty" protobuf:"varint,5,opt,name=unavailableReplicas"`
 
-    // Total number of terminating pods (have .metadata.deletionTimestamp) targeted by this deployment.
+    // Total number of terminating pods (have a non-null .metadata.deletionTimestamp) targeted by this deployment.
     //
     // This is an alpha field. Enable DeploymentPodReplacementPolicy to be able to use this field.
     // +optional
-    TerminatingReplicas int32 `json:"terminatingReplicas,omitempty" protobuf:"varint,9,opt,name=terminatingReplicas"`
+    TerminatingReplicas *int32 `json:"terminatingReplicas,omitempty" protobuf:"varint,9,opt,name=terminatingReplicas"`
     ...
 }
 ```
@@ -929,6 +949,7 @@ deployment and replicaset controllers.
 - 2023-05-01: First version of the KEP opened (https://github.com/kubernetes/enhancements/pull/3974).
 - 2023-12-12: Second version of the KEP opened (https://github.com/kubernetes/enhancements/pull/4357).
 - 2024-29-05: Added a Deployment Scaling Changes and a New Annotation for ReplicaSets section (https://github.com/kubernetes/enhancements/pull/4670).
+- 2024-22-11: Added a Deployment Completion and Progress Changes section (https://github.com/kubernetes/enhancements/pull/4976).
 
 ## Drawbacks
 

keps/sig-apps/3973-consider-terminating-pods-deployment/kep.yaml

@@ -22,11 +22,11 @@ stage: alpha
 # The most recent milestone for which work toward delivery of this KEP has been
 # done. This can be the current (upcoming) milestone, if it is being actively
 # worked on.
-latest-milestone: "v1.32"
+latest-milestone: "v1.33"
 
 # The milestone at which this feature was, or is targeted to be, at each stage.
 milestone:
-  alpha: "v1.32"
+  alpha: "v1.33"
   beta: ""
   stable: ""
 

keps/sig-apps/4026-crojob-scheduled-timestamp-annotation/kep.yaml

@@ -5,7 +5,7 @@ authors:
   - "@soltysh"
 owning-sig: sig-apps
 participating-sigs:
-status: implementable
+status: implemented
 creation-date: 2023-06-06
 reviewers:
   - "@soltysh"

keps/sig-auth/4193-bound-service-account-token-improvements/README.md

@@ -724,6 +724,7 @@ For example, attempting to issue a node bound token, or attempting to authentica
 * `ServiceAccountTokenJTI`, `ServiceAccountTokenNodeBindingValidation` and `ServiceAccountTokenPodNodeInfo` promoted to beta for v1.30 release
 * Promoted `ServiceAccountTokenNodeBinding` promoted to beta for v1.31 release
 * Promoted `ServiceAccountTokenJTI`, `ServiceAccountTokenPodNodeInfo`, `ServiceAccountTokenNodeBindingValidation` to stable for v1.32 release
+* Promoted `ServiceAccountTokenNodeBinding` to stable for v1.33 release
 
 <!--
 Major milestones in the lifecycle of a KEP should be tracked in this section.