Skip to content

Commit fbbc3a5

Browse files
d80tb7d80tb7
committed
Remove Market Based Scheduling
1 parent 7136ee9 commit fbbc3a5

40 files changed

+621
-2526
lines changed

‎internal/common/eventutil/eventutil.go‎

Lines changed: 3 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -160,13 +160,6 @@ func ApiJobFromLogSubmitJob(ownerId string, groups []string, queueName string, j
160160
podSpecs = k8sPodSpecs
161161
}
162162

163-
var priceInfo *api.ExperimentalPriceInfo
164-
if e.ExperimentalPriceInfo != nil {
165-
priceInfo = &api.ExperimentalPriceInfo{
166-
BidPrice: e.ExperimentalPriceInfo.BidPrice,
167-
}
168-
}
169-
170163
return &api.Job{
171164
Id: e.JobId,
172165
ClientId: e.DeduplicationId,
@@ -177,10 +170,9 @@ func ApiJobFromLogSubmitJob(ownerId string, groups []string, queueName string, j
177170
Labels: e.ObjectMeta.Labels,
178171
Annotations: e.ObjectMeta.Annotations,
179172

180-
K8SIngress: k8sIngresses,
181-
K8SService: k8sServices,
182-
ExperimentalPriceInfo: priceInfo,
183-
Priority: float64(e.Priority),
173+
K8SIngress: k8sIngresses,
174+
K8SService: k8sServices,
175+
Priority: float64(e.Priority),
184176

185177
PodSpec: podSpec,
186178
PodSpecs: podSpecs,

‎internal/common/metrics/domain.go‎

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -19,15 +19,13 @@ type QueueMetrics struct {
1919
PriorityClass string
2020
Resources ResourceMetrics
2121
Durations *FloatMetrics
22-
BidPrices *FloatMetrics
2322
}
2423

2524
type QueueMetricsRecorder struct {
2625
Pool string
2726
PriorityClass string
2827
resourceRecorder *ResourceMetricsRecorder
2928
durationRecorder *FloatMetricsRecorder
30-
bidPriceRecorder *FloatMetricsRecorder
3129
}
3230

3331
type JobMetricsRecorder struct {
@@ -48,11 +46,6 @@ func (r *JobMetricsRecorder) RecordResources(pool string, priorityClass string,
4846
recorder.resourceRecorder.Record(resources)
4947
}
5048

51-
func (r *JobMetricsRecorder) RecordBidPrice(pool string, priorityClass string, price float64) {
52-
recorder := r.getOrCreateRecorder(pool, priorityClass)
53-
recorder.bidPriceRecorder.Record(price)
54-
}
55-
5649
func (r *JobMetricsRecorder) Metrics() []*QueueMetrics {
5750
result := make([]*QueueMetrics, 0, len(r.recordersByPoolAndPriorityClass))
5851
for _, v := range r.recordersByPoolAndPriorityClass {
@@ -61,7 +54,6 @@ func (r *JobMetricsRecorder) Metrics() []*QueueMetrics {
6154
PriorityClass: v.PriorityClass,
6255
Resources: v.resourceRecorder.GetMetrics(),
6356
Durations: v.durationRecorder.GetMetrics(),
64-
BidPrices: v.bidPriceRecorder.GetMetrics(),
6557
})
6658
}
6759
return result
@@ -76,7 +68,6 @@ func (r *JobMetricsRecorder) getOrCreateRecorder(pool string, priorityClass stri
7668
PriorityClass: priorityClass,
7769
resourceRecorder: NewResourceMetricsRecorder(),
7870
durationRecorder: NewDefaultJobDurationMetricsRecorder(),
79-
bidPriceRecorder: NewFloatMetricsRecorder(),
8071
}
8172
r.recordersByPoolAndPriorityClass[recorderKey] = qmr
8273
}

‎internal/common/metrics/scheduler_metrics.go‎

Lines changed: 0 additions & 74 deletions
Original file line numberDiff line numberDiff line change
@@ -24,48 +24,6 @@ var QueueDistinctSchedulingKeysDesc = prometheus.NewDesc(
2424
nil,
2525
)
2626

27-
var MinQueuePriceQueuedDesc = prometheus.NewDesc(
28-
MetricPrefix+"queue_price_queued_min",
29-
"Minimum price of queued jobs",
30-
[]string{"pool", "priorityClass", "queue"},
31-
nil,
32-
)
33-
34-
var MaxQueuePriceQueuedDesc = prometheus.NewDesc(
35-
MetricPrefix+"queue_price_queued_max",
36-
"Maximum price of queued jobs",
37-
[]string{"pool", "priorityClass", "queue"},
38-
nil,
39-
)
40-
41-
var MedianQueuePriceQueuedDesc = prometheus.NewDesc(
42-
MetricPrefix+"queue_price_queued_median",
43-
"Median price of queued jobs",
44-
[]string{"pool", "priorityClass", "queue"},
45-
nil,
46-
)
47-
48-
var MinQueuePriceRunningDesc = prometheus.NewDesc(
49-
MetricPrefix+"queue_price_running_min",
50-
"Minimum price of running jobs",
51-
[]string{"pool", "priorityClass", "queue"},
52-
nil,
53-
)
54-
55-
var MaxQueuePriceRunningDesc = prometheus.NewDesc(
56-
MetricPrefix+"queue_price_running_max",
57-
"Maximum price of running jobs",
58-
[]string{"pool", "priorityClass", "queue"},
59-
nil,
60-
)
61-
62-
var MedianQueuePriceRunningDesc = prometheus.NewDesc(
63-
MetricPrefix+"queue_price_running_median",
64-
"Median price of running jobs",
65-
[]string{"pool", "priorityClass", "queue"},
66-
nil,
67-
)
68-
6927
var QueueResourcesDesc = prometheus.NewDesc(
7028
MetricPrefix+"queue_resource_queued",
7129
"Resource required by queued jobs",
@@ -294,10 +252,6 @@ func CollectQueueMetrics(queueCounts map[string]int, queueDistinctSchedulingKeyC
294252
metrics = append(metrics, NewMedianQueueDuration(queueDurations.GetMedian(), m.Pool, m.PriorityClass, q))
295253
}
296254

297-
metrics = append(metrics, NewMinQueuePriceQueuedMetric(m.BidPrices.GetMin(), m.Pool, m.PriorityClass, q))
298-
metrics = append(metrics, NewMaxQueuePriceQueuedMetric(m.BidPrices.GetMax(), m.Pool, m.PriorityClass, q))
299-
metrics = append(metrics, NewMedianQueuePriceQueuedMetric(m.BidPrices.GetMedian(), m.Pool, m.PriorityClass, q))
300-
301255
// Sort the keys so we get a predictable output order
302256
resources := maps.Keys(m.Resources)
303257
slices.Sort(resources)
@@ -323,10 +277,6 @@ func CollectQueueMetrics(queueCounts map[string]int, queueDistinctSchedulingKeyC
323277
metrics = append(metrics, NewMedianJobRunDuration(runningJobDurations.GetMedian(), m.Pool, m.PriorityClass, q))
324278
}
325279

326-
metrics = append(metrics, NewMinQueuePriceRunningMetric(m.BidPrices.GetMin(), m.Pool, m.PriorityClass, q))
327-
metrics = append(metrics, NewMaxQueuePriceRunningMetric(m.BidPrices.GetMax(), m.Pool, m.PriorityClass, q))
328-
metrics = append(metrics, NewMedianQueuePriceRunningMetric(m.BidPrices.GetMedian(), m.Pool, m.PriorityClass, q))
329-
330280
// Sort the keys so we get a predicatable output order
331281
resources := maps.Keys(m.Resources)
332282
slices.Sort(resources)
@@ -448,30 +398,6 @@ func NewQueuePriorityMetric(value float64, queue string) prometheus.Metric {
448398
return prometheus.MustNewConstMetric(QueuePriorityDesc, prometheus.GaugeValue, value, queue, queue)
449399
}
450400

451-
func NewMinQueuePriceQueuedMetric(value float64, pool string, priorityClass string, queue string) prometheus.Metric {
452-
return prometheus.MustNewConstMetric(MinQueuePriceQueuedDesc, prometheus.GaugeValue, value, pool, priorityClass, queue)
453-
}
454-
455-
func NewMaxQueuePriceQueuedMetric(value float64, pool string, priorityClass string, queue string) prometheus.Metric {
456-
return prometheus.MustNewConstMetric(MaxQueuePriceQueuedDesc, prometheus.GaugeValue, value, pool, priorityClass, queue)
457-
}
458-
459-
func NewMedianQueuePriceQueuedMetric(value float64, pool string, priorityClass string, queue string) prometheus.Metric {
460-
return prometheus.MustNewConstMetric(MedianQueuePriceQueuedDesc, prometheus.GaugeValue, value, pool, priorityClass, queue)
461-
}
462-
463-
func NewMinQueuePriceRunningMetric(value float64, pool string, priorityClass string, queue string) prometheus.Metric {
464-
return prometheus.MustNewConstMetric(MinQueuePriceRunningDesc, prometheus.GaugeValue, value, pool, priorityClass, queue)
465-
}
466-
467-
func NewMaxQueuePriceRunningMetric(value float64, pool string, priorityClass string, queue string) prometheus.Metric {
468-
return prometheus.MustNewConstMetric(MaxQueuePriceRunningDesc, prometheus.GaugeValue, value, pool, priorityClass, queue)
469-
}
470-
471-
func NewMedianQueuePriceRunningMetric(value float64, pool string, priorityClass string, queue string) prometheus.Metric {
472-
return prometheus.MustNewConstMetric(MedianQueuePriceRunningDesc, prometheus.GaugeValue, value, pool, priorityClass, queue)
473-
}
474-
475401
func NewQueueLabelsMetric(queue string, labels map[string]string) prometheus.Metric {
476402
metricLabels := make([]string, 0, len(labels)+len(queueLabelDefaultLabels))
477403
values := make([]string, 0, len(labels)+len(queueLabelDefaultLabels))

‎internal/scheduler/configuration/configuration.go‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -281,7 +281,6 @@ type PoolConfig struct {
281281
Name string `validate:"required"`
282282
AwayPools []string
283283
ProtectedFractionOfFairShare *float64
284-
MarketDriven bool
285284
ExperimentalProtectUncappedAdjustedFairShare bool
286285
}
287286

‎internal/scheduler/database/job_repository.go‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -114,7 +114,6 @@ func (r *PostgresJobRepository) FetchInitialJobs(ctx *armadacontext.Context) ([]
114114
JobSet: row.JobSet,
115115
Queue: row.Queue,
116116
Priority: row.Priority,
117-
BidPrice: row.BidPrice,
118117
Submitted: row.Submitted,
119118
Validated: row.Validated,
120119
Queued: row.Queued,
@@ -231,7 +230,6 @@ func (r *PostgresJobRepository) FetchJobUpdates(ctx *armadacontext.Context, jobS
231230
JobSet: row.JobSet,
232231
Queue: row.Queue,
233232
Priority: row.Priority,
234-
BidPrice: row.BidPrice,
235233
Submitted: row.Submitted,
236234
Validated: row.Validated,
237235
Queued: row.Queued,

‎internal/scheduler/database/migrations/021_remove_price.sql‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
ALTER TABLE jobs DROP COLUMN bid_price;

‎internal/scheduler/database/models.go‎

Lines changed: 0 additions & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎internal/scheduler/database/query.sql.go‎

Lines changed: 8 additions & 18 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎internal/scheduler/database/query/query.sql‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,10 @@ SELECT * FROM jobs WHERE serial > $1 ORDER BY serial LIMIT $2;
55
SELECT job_id FROM jobs;
66

77
-- name: SelectInitialJobs :many
8-
SELECT job_id, job_set, queue, priority, bid_price, submitted, queued, queued_version, validated, cancel_requested, cancel_user, cancel_by_jobset_requested, cancelled, succeeded, failed, scheduling_info, scheduling_info_version, pools, serial FROM jobs WHERE serial > $1 AND cancelled = 'false' AND succeeded = 'false' and failed = 'false' ORDER BY serial LIMIT $2;
8+
SELECT job_id, job_set, queue, priority, submitted, queued, queued_version, validated, cancel_requested, cancel_user, cancel_by_jobset_requested, cancelled, succeeded, failed, scheduling_info, scheduling_info_version, pools, serial FROM jobs WHERE serial > $1 AND cancelled = 'false' AND succeeded = 'false' and failed = 'false' ORDER BY serial LIMIT $2;
99

1010
-- name: SelectUpdatedJobs :many
11-
SELECT job_id, job_set, queue, priority, bid_price, submitted, queued, queued_version, validated, cancel_requested, cancel_user, cancel_by_jobset_requested, cancelled, succeeded, failed, scheduling_info, scheduling_info_version, pools, serial FROM jobs WHERE serial > $1 ORDER BY serial LIMIT $2;
11+
SELECT job_id, job_set, queue, priority, submitted, queued, queued_version, validated, cancel_requested, cancel_user, cancel_by_jobset_requested, cancelled, succeeded, failed, scheduling_info, scheduling_info_version, pools, serial FROM jobs WHERE serial > $1 ORDER BY serial LIMIT $2;
1212

1313
-- name: UpdateJobPriorityByJobSet :exec
1414
UPDATE jobs SET priority = $1 WHERE job_set = $2 and queue = $3 and cancelled = false and succeeded = false and failed = false;

0 commit comments

Comments
 (0)