add preemptable candidates (#493)

* add preemptable candidates

* Revert "add preemptable candidates"

This reverts commit da08c4a.

* Revert "Revert "add preemptable candidates""

This reverts commit 4d0da32.

* address review

Author: asm582

pkg/controller/queuejob/queuejob_controller_ex.go

@@ -955,9 +955,6 @@ func (qjm *XController) getAggregatedAvailableResourcesPriority(unallocatedClust
 					continue
 				}
 
-				preemptableAWs[value.Status.SystemPriority] = append(preemptableAWs[value.Status.SystemPriority], queueJobKey)
-				preemptableAWsMap[queueJobKey] = value
-				klog.V(10).Infof("[getAggAvaiResPri] %s: Added %s to candidate preemptable job with priority %f.", time.Now().String(), value.Name, value.Status.SystemPriority)
 			}
 
 			err := qjm.qjobResControls[arbv1.ResourceTypePod].UpdateQueueJobStatus(value)
@@ -969,6 +966,8 @@ func (qjm *XController) getAggregatedAvailableResourcesPriority(unallocatedClust
 			klog.V(10).Infof("[getAggAvaiResPri] total resources consumed by Appwrapper %v when lower priority compared to target are %v", value.Name, totalResource)
 			preemptable = preemptable.Add(totalResource)
 			klog.V(6).Infof("[getAggAvaiResPri] %s proirity %v is lower target priority %v reclaiming total preemptable resources %v", value.Name, value.Status.SystemPriority, targetpr, totalResource)
+			queueJobKey, _ := GetQueueJobKey(value)
+			addPreemptableAWs(preemptableAWs, value, queueJobKey, preemptableAWsMap)
 			continue
 		} else if qjm.isDispatcher {
 			// Dispatcher job does not currently track pod states.  This is
@@ -1017,6 +1016,12 @@ func (qjm *XController) getAggregatedAvailableResourcesPriority(unallocatedClust
 	return r, proposedPremptions
 }
 
+func addPreemptableAWs(preemptableAWs map[float64][]string, value *arbv1.AppWrapper, queueJobKey string, preemptableAWsMap map[string]*arbv1.AppWrapper) {
+	preemptableAWs[value.Status.SystemPriority] = append(preemptableAWs[value.Status.SystemPriority], queueJobKey)
+	preemptableAWsMap[queueJobKey] = value
+	klog.V(10).Infof("[getAggAvaiResPri] %s: Added %s to candidate preemptable job with priority %f.", time.Now().String(), value.Name, value.Status.SystemPriority)
+}
+
 func (qjm *XController) chooseAgent(qj *arbv1.AppWrapper) string {
 
 	qjAggrResources := qjm.GetAggregatedResources(qj)