Fix: Sidecar crashes during initialization lead to pods not being able to initialize [other implementation] (#570)

* PublishNotReadyAddresses & WithReadinessByContainer

* fix style

* Canonize

* Return immediately if requested container is not ready

Author: ilyaibraev

pkg/components/master.go

@@ -45,6 +45,7 @@ func buildMasterOptions(resource *ytv1.Ytsaurus) []Option {
 			ContainerPort: consts.MasterRPCPort,
 			Protocol:      corev1.ProtocolTCP,
 		}),
+		WithReadinessByContainer(consts.YTServerContainerName),
 	}
 
 	if resource.Spec.PrimaryMasters.HydraPersistenceUploader != nil && resource.Spec.PrimaryMasters.HydraPersistenceUploader.Image != nil {

pkg/components/server.go

@@ -64,6 +64,7 @@ type serverImpl struct {
 
 	readinessProbePort     intstr.IntOrString
 	readinessProbeHTTPPath string
+	readinessByContainers  []string
 }
 
 func newServer(
@@ -191,6 +192,7 @@ func newServerConfigured(
 
 		readinessProbePort:     opts.readinessProbeEndpointPort,
 		readinessProbeHTTPPath: opts.readinessProbeEndpointPath,
+		readinessByContainers:  opts.readinessByContainers,
 	}
 }
 
@@ -291,7 +293,7 @@ func (s *serverImpl) needUpdate() bool {
 }
 
 func (s *serverImpl) arePodsReady(ctx context.Context) bool {
-	return s.statefulSet.ArePodsReady(ctx, int(s.instanceSpec.InstanceCount), s.instanceSpec.MinReadyInstanceCount)
+	return s.statefulSet.ArePodsReady(ctx, int(s.instanceSpec.InstanceCount), s.instanceSpec.MinReadyInstanceCount, s.readinessByContainers)
 }
 
 func (s *serverImpl) buildStatefulSet() *appsv1.StatefulSet {

pkg/components/serveroptions.go

@@ -12,6 +12,8 @@ type options struct {
 	readinessProbeEndpointPath string
 
 	sidecarImages map[string]string
+
+	readinessByContainers []string
 }
 
 type Option func(opts *options)
@@ -42,3 +44,9 @@ func WithSidecarImage(name, image string) Option {
 		opts.sidecarImages[name] = image
 	}
 }
+
+func WithReadinessByContainer(name string) Option {
+	return func(opts *options) {
+		opts.readinessByContainers = append(opts.readinessByContainers, name)
+	}
+}

pkg/resources/headless_service.go

@@ -25,8 +25,9 @@ func NewHeadlessService(name string, labeller *labeller2.Labeller, apiProxy apip
 func (s *HeadlessService) Build() *corev1.Service {
 	s.newObject.ObjectMeta = s.labeller.GetObjectMeta(s.name)
 	s.newObject.Spec = corev1.ServiceSpec{
-		ClusterIP: "None",
-		Selector:  s.labeller.GetSelectorLabelMap(),
+		ClusterIP:                "None",
+		Selector:                 s.labeller.GetSelectorLabelMap(),
+		PublishNotReadyAddresses: true,
 	}
 
 	return s.newObject

pkg/resources/statefulset.go

@@ -86,7 +86,23 @@ func (s *StatefulSet) ArePodsRemoved(ctx context.Context) bool {
 	return true
 }
 
-func (s *StatefulSet) ArePodsReady(ctx context.Context, instanceCount int, minReadyInstanceCount *int) bool {
+func checkReadinessByContainers(pod corev1.Pod, byContainerNames []string) bool {
+	found := 0
+	for _, containerNameToCheck := range byContainerNames {
+		for _, containerStatus := range pod.Status.ContainerStatuses {
+			if containerStatus.Name != containerNameToCheck {
+				continue
+			}
+			if !containerStatus.Ready {
+				return false
+			}
+			found++
+		}
+	}
+	return found == len(byContainerNames)
+}
+
+func (s *StatefulSet) ArePodsReady(ctx context.Context, instanceCount int, minReadyInstanceCount *int, byContainerNames []string) bool {
 	logger := log.FromContext(ctx)
 	podList := s.getPods(ctx)
 	if podList == nil {
@@ -100,7 +116,13 @@ func (s *StatefulSet) ArePodsReady(ctx context.Context, instanceCount int, minRe
 
 	readyInstanceCount := 0
 	for _, pod := range podList.Items {
-		if pod.Status.Phase != corev1.PodRunning {
+		var ready bool
+		if len(byContainerNames) > 0 {
+			ready = checkReadinessByContainers(pod, byContainerNames)
+		} else {
+			ready = pod.Status.Phase == corev1.PodRunning
+		}
+		if !ready {
 			logger.Info("pod is not yet running", "podName", pod.Name, "phase", pod.Status.Phase)
 		} else {
 			readyInstanceCount += 1

test/r8r/canondata/Components reconciler Minimal Test/Service discovery.yaml

@@ -22,6 +22,7 @@ metadata:
   resourceVersion: "1"
 spec:
   clusterIP: None
+  publishNotReadyAddresses: true
   selector:
     yt_component: test-ytsaurus-yt-discovery
 status:

test/r8r/canondata/Components reconciler Minimal Test/Service http-proxies.yaml

@@ -22,6 +22,7 @@ metadata:
   resourceVersion: "1"
 spec:
   clusterIP: None
+  publishNotReadyAddresses: true
   selector:
     yt_component: test-ytsaurus-yt-http-proxy
 status:

test/r8r/canondata/Components reconciler Minimal Test/Service masters.yaml

@@ -22,6 +22,7 @@ metadata:
   resourceVersion: "1"
 spec:
   clusterIP: None
+  publishNotReadyAddresses: true
   selector:
     yt_component: test-ytsaurus-yt-master
 status:

test/r8r/canondata/Components reconciler With CRI job environment - CRI-O Test/Service discovery.yaml

@@ -22,6 +22,7 @@ metadata:
   resourceVersion: "1"
 spec:
   clusterIP: None
+  publishNotReadyAddresses: true
   selector:
     yt_component: test-ytsaurus-yt-discovery
 status:

test/r8r/canondata/Components reconciler With CRI job environment - CRI-O Test/Service exec-nodes.yaml

@@ -22,6 +22,7 @@ metadata:
   resourceVersion: "1"
 spec:
   clusterIP: None
+  publishNotReadyAddresses: true
   selector:
     yt_component: test-ytsaurus-yt-exec-node
 status: