CARRY: Use temp folder to store progress file

Author: sutaakar

manifests/base/crds/kubeflow.org_jaxjobs.yaml

@@ -7761,6 +7761,11 @@ spec:
               Most recently observed status of the JAXJob.
               Read-only (modified by the system).
             properties:
+              completionPercentage:
+                description: |-
+                  CompletionPercentage is the percentage of the job that has completed from 0.0 to 100.0,
+                  formatted as a string with one decimal place (e.g., "45.2").
+                type: string
               completionTime:
                 description: |-
                   Represents time when the job was completed. It is not guaranteed to

manifests/base/crds/kubeflow.org_mpijobs.yaml

@@ -7771,6 +7771,11 @@ spec:
             description: JobStatus represents the current observed state of the training
               Job.
             properties:
+              completionPercentage:
+                description: |-
+                  CompletionPercentage is the percentage of the job that has completed from 0.0 to 100.0,
+                  formatted as a string with one decimal place (e.g., "45.2").
+                type: string
               completionTime:
                 description: |-
                   Represents time when the job was completed. It is not guaranteed to

manifests/base/crds/kubeflow.org_paddlejobs.yaml

@@ -8254,6 +8254,11 @@ spec:
               Most recently observed status of the PaddleJob.
               Read-only (modified by the system).
             properties:
+              completionPercentage:
+                description: |-
+                  CompletionPercentage is the percentage of the job that has completed from 0.0 to 100.0,
+                  formatted as a string with one decimal place (e.g., "45.2").
+                type: string
               completionTime:
                 description: |-
                   Represents time when the job was completed. It is not guaranteed to

manifests/base/crds/kubeflow.org_pytorchjobs.yaml

@@ -8292,9 +8292,9 @@ spec:
               Read-only (modified by the system).
             properties:
               completionPercentage:
-                description: CompletionPercentage is the percentage of the job that
-                  has completed from 0.0 to 100.0, formatted as a string with one
-                  decimal place (e.g., "45.2").
+                description: |-
+                  CompletionPercentage is the percentage of the job that has completed from 0.0 to 100.0,
+                  formatted as a string with one decimal place (e.g., "45.2").
                 type: string
               completionTime:
                 description: |-

manifests/base/crds/kubeflow.org_tfjobs.yaml

@@ -7771,6 +7771,11 @@ spec:
               Populated by the system.
               Read-only.
             properties:
+              completionPercentage:
+                description: |-
+                  CompletionPercentage is the percentage of the job that has completed from 0.0 to 100.0,
+                  formatted as a string with one decimal place (e.g., "45.2").
+                type: string
               completionTime:
                 description: |-
                   Represents time when the job was completed. It is not guaranteed to

manifests/base/crds/kubeflow.org_xgboostjobs.yaml

@@ -7753,6 +7753,11 @@ spec:
             description: JobStatus represents the current observed state of the training
               Job.
             properties:
+              completionPercentage:
+                description: |-
+                  CompletionPercentage is the percentage of the job that has completed from 0.0 to 100.0,
+                  formatted as a string with one decimal place (e.g., "45.2").
+                type: string
               completionTime:
                 description: |-
                   Represents time when the job was completed. It is not guaranteed to

pkg/apis/kubeflow.org/v1/zz_generated.openapi.go

@@ -121,6 +121,12 @@ func setPodEnv(obj interface{}, podTemplateSpec *corev1.PodTemplateSpec, rtype,
 					Value: strconv.Itoa(int(totalReplicas)),
 				})
 		}
+
+		// Set the training progress file path.
+		podTemplateSpec.Spec.Containers[i].Env = append(podTemplateSpec.Spec.Containers[i].Env, corev1.EnvVar{
+			Name:  EnvTrainingProgressFilePath,
+			Value: GetProgressFilePath(pytorchjob),
+		})
 	}
 
 	return nil

pkg/client/applyconfiguration/kubeflow.org/v1/jobstatus.go

@@ -191,15 +191,7 @@ func (r *PyTorchJobReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 		return ctrl.Result{}, err
 	}
 
-	jobIsRunning := false
-	for _, condition := range pytorchjob.Status.Conditions {
-		if condition.Type == kubeflowv1.JobRunning && condition.Status == corev1.ConditionTrue {
-			jobIsRunning = true
-			break
-		}
-	}
-
-	if jobIsRunning {
+	if commonutil.IsRunning(pytorchjob.Status) {
 		if content, err := r.readCompletionPercentageFromPod(pytorchjob); err == nil {
 			if percentage, parseErr := r.parseCompletionPercentage(content); parseErr == nil {
 				// Assuming your PyTorchJobStatus has a CompletionPercentage field.
@@ -219,12 +211,12 @@ func (r *PyTorchJobReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 		} else {
 			logrus.Debugf("Failed to read completion percentage from rank-0 pod for PyTorchJob %s: %v", pytorchjob.Name, err)
 		}
-		
+
 		// Return a short requeue interval for running jobs
 		// TODO instead of hard coding the requeue interval we could make this configurable
 		return ctrl.Result{RequeueAfter: 30 * time.Second}, nil
 	}
-	
+
 	t, err := util.DurationUntilExpireTime(&pytorchjob.Spec.RunPolicy, pytorchjob.Status)
 	if err != nil {
 		logrus.Warnf("Reconcile PyTorchJob error %v", err)
@@ -527,7 +519,9 @@ func (r *PyTorchJobReconciler) execInPod(pod *corev1.Pod, containerName string,
 	}
 
 	var stdout, stderr bytes.Buffer
-	err = executor.StreamWithContext(context.Background(), remotecommand.StreamOptions{
+	ctx, cancel := context.WithTimeoutCause(context.Background(), 10*time.Second, fmt.Errorf("pod execution timed out"))
+	defer cancel()
+	err = executor.StreamWithContext(ctx, remotecommand.StreamOptions{
 		Stdout: &stdout,
 		Stderr: &stderr,
 	})
@@ -576,16 +570,13 @@ func (r *PyTorchJobReconciler) readCompletionPercentageFromPod(pytorchjob *kubef
 		return "", fmt.Errorf("rank-0 pod %s is not in running state: %s", rankZeroPod.Name, rankZeroPod.Status.Phase)
 	}
 
-	// Get the container name (use default PyTorch container name)
-	containerName := kubeflowv1.PyTorchJobDefaultContainerName
-	if len(rankZeroPod.Spec.Containers) > 0 {
-		containerName = rankZeroPod.Spec.Containers[0].Name
-	}
+	// Get the container name
+	containerName := rankZeroPod.Spec.Containers[0].Name
 
 	// Read the progress.json file from /mnt/checkpoints - /var/run is not accessible by non-root user
 	// TODO we could have the user add the file path in an annotation instead of hardcoding it here
 	// later we could update the CRD spec to allow for checkpoint config
-	progressFilePath := "/mnt/checkpoints/progress.json"
+	progressFilePath := GetProgressFilePath(pytorchjob)
 	catCommand := []string{"cat", progressFilePath}
 	content, err := r.execInPod(rankZeroPod, containerName, catCommand)
 	if err != nil {
@@ -600,7 +591,7 @@ func (r *PyTorchJobReconciler) parseCompletionPercentage(content string) (string
 	var progress ProgressData
 
 	if err := json.Unmarshal([]byte(content), &progress); err != nil {
-		return "", fmt.Errorf("failed to parse JSON: %v", err)
+		return "", fmt.Errorf("failed to parse JSON from content '%s': %v", content, err)
 	}
 
 	// Extract current and total steps