Skip to content

Commit fd7d528

Browse files
derklingrafaeljw
authored andcommitted
cpufreq: schedutil: Cleanup and document iowait boost
The iowait boosting code has been recently updated to add a progressive boosting behavior which allows to be less aggressive in boosting tasks doing only sporadic IO operations, thus being more energy efficient for example on mobile platforms. The current code is now however a bit convoluted. Some functionalities (e.g. iowait boost reset) are replicated in different paths and their documentation is slightly misaligned. Let's cleanup the code by consolidating all the IO wait boosting related functionality within within few dedicated functions and better define their role: - sugov_iowait_boost: set/increase the IO wait boost of a CPU - sugov_iowait_apply: apply/reduce the IO wait boost of a CPU Both these two function are used at every sugov update and they make use of a unified IO wait boost reset policy provided by: - sugov_iowait_reset: reset/disable the IO wait boost of a CPU if a CPU is not updated for more then one tick This makes possible a cleaner and more self-contained design for the IO wait boosting code since the rest of the sugov update routines, both for single and shared frequency domains, follow the same template: /* Configure IO boost, if required */ sugov_iowait_boost() /* Return here if freq change is in progress or throttled */ /* Collect and aggregate utilization information */ sugov_get_util() sugov_aggregate_util() /* * Add IO boost, if currently enabled, on top of the aggregated * utilization value */ sugov_iowait_apply() As a extra bonus, let's also add the documentation for the new functions and better align the in-code documentation. Signed-off-by: Patrick Bellasi <[email protected]> Reviewed-by: Joel Fernandes (Google) <[email protected]> Acked-by: Viresh Kumar <[email protected]> Acked-by: Peter Zijlstra (Intel) <[email protected]> Signed-off-by: Rafael J. Wysocki <[email protected]>
1 parent 295f1a9 commit fd7d528

File tree

1 file changed

+107
-45
lines changed

1 file changed

+107
-45
lines changed

kernel/sched/cpufreq_schedutil.c

Lines changed: 107 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ struct sugov_cpu {
5151
bool iowait_boost_pending;
5252
unsigned int iowait_boost;
5353
unsigned int iowait_boost_max;
54-
u64 last_update;
54+
u64 last_update;
5555

5656
/* The fields below are only needed when sharing a policy: */
5757
unsigned long util_cfs;
@@ -196,55 +196,133 @@ static unsigned long sugov_aggregate_util(struct sugov_cpu *sg_cpu)
196196
return min(util, sg_cpu->max);
197197
}
198198

199-
static void sugov_set_iowait_boost(struct sugov_cpu *sg_cpu, u64 time, unsigned int flags)
199+
/**
200+
* sugov_iowait_reset() - Reset the IO boost status of a CPU.
201+
* @sg_cpu: the sugov data for the CPU to boost
202+
* @time: the update time from the caller
203+
* @set_iowait_boost: true if an IO boost has been requested
204+
*
205+
* The IO wait boost of a task is disabled after a tick since the last update
206+
* of a CPU. If a new IO wait boost is requested after more then a tick, then
207+
* we enable the boost starting from the minimum frequency, which improves
208+
* energy efficiency by ignoring sporadic wakeups from IO.
209+
*/
210+
static bool sugov_iowait_reset(struct sugov_cpu *sg_cpu, u64 time,
211+
bool set_iowait_boost)
200212
{
201-
/* Clear iowait_boost if the CPU apprears to have been idle. */
202-
if (sg_cpu->iowait_boost) {
203-
s64 delta_ns = time - sg_cpu->last_update;
213+
s64 delta_ns = time - sg_cpu->last_update;
204214

205-
if (delta_ns > TICK_NSEC) {
206-
sg_cpu->iowait_boost = 0;
207-
sg_cpu->iowait_boost_pending = false;
208-
}
209-
}
215+
/* Reset boost only if a tick has elapsed since last request */
216+
if (delta_ns <= TICK_NSEC)
217+
return false;
210218

211-
if (flags & SCHED_CPUFREQ_IOWAIT) {
212-
if (sg_cpu->iowait_boost_pending)
213-
return;
219+
sg_cpu->iowait_boost = set_iowait_boost
220+
? sg_cpu->sg_policy->policy->min : 0;
221+
sg_cpu->iowait_boost_pending = set_iowait_boost;
214222

215-
sg_cpu->iowait_boost_pending = true;
223+
return true;
224+
}
216225

217-
if (sg_cpu->iowait_boost) {
218-
sg_cpu->iowait_boost <<= 1;
219-
if (sg_cpu->iowait_boost > sg_cpu->iowait_boost_max)
220-
sg_cpu->iowait_boost = sg_cpu->iowait_boost_max;
221-
} else {
222-
sg_cpu->iowait_boost = sg_cpu->sg_policy->policy->min;
223-
}
226+
/**
227+
* sugov_iowait_boost() - Updates the IO boost status of a CPU.
228+
* @sg_cpu: the sugov data for the CPU to boost
229+
* @time: the update time from the caller
230+
* @flags: SCHED_CPUFREQ_IOWAIT if the task is waking up after an IO wait
231+
*
232+
* Each time a task wakes up after an IO operation, the CPU utilization can be
233+
* boosted to a certain utilization which doubles at each "frequent and
234+
* successive" wakeup from IO, ranging from the utilization of the minimum
235+
* OPP to the utilization of the maximum OPP.
236+
* To keep doubling, an IO boost has to be requested at least once per tick,
237+
* otherwise we restart from the utilization of the minimum OPP.
238+
*/
239+
static void sugov_iowait_boost(struct sugov_cpu *sg_cpu, u64 time,
240+
unsigned int flags)
241+
{
242+
bool set_iowait_boost = flags & SCHED_CPUFREQ_IOWAIT;
243+
244+
/* Reset boost if the CPU appears to have been idle enough */
245+
if (sg_cpu->iowait_boost &&
246+
sugov_iowait_reset(sg_cpu, time, set_iowait_boost))
247+
return;
248+
249+
/* Boost only tasks waking up after IO */
250+
if (!set_iowait_boost)
251+
return;
252+
253+
/* Ensure boost doubles only one time at each request */
254+
if (sg_cpu->iowait_boost_pending)
255+
return;
256+
sg_cpu->iowait_boost_pending = true;
257+
258+
/* Double the boost at each request */
259+
if (sg_cpu->iowait_boost) {
260+
sg_cpu->iowait_boost <<= 1;
261+
if (sg_cpu->iowait_boost > sg_cpu->iowait_boost_max)
262+
sg_cpu->iowait_boost = sg_cpu->iowait_boost_max;
263+
return;
224264
}
265+
266+
/* First wakeup after IO: start with minimum boost */
267+
sg_cpu->iowait_boost = sg_cpu->sg_policy->policy->min;
225268
}
226269

227-
static void sugov_iowait_boost(struct sugov_cpu *sg_cpu, unsigned long *util,
228-
unsigned long *max)
270+
/**
271+
* sugov_iowait_apply() - Apply the IO boost to a CPU.
272+
* @sg_cpu: the sugov data for the cpu to boost
273+
* @time: the update time from the caller
274+
* @util: the utilization to (eventually) boost
275+
* @max: the maximum value the utilization can be boosted to
276+
*
277+
* A CPU running a task which woken up after an IO operation can have its
278+
* utilization boosted to speed up the completion of those IO operations.
279+
* The IO boost value is increased each time a task wakes up from IO, in
280+
* sugov_iowait_apply(), and it's instead decreased by this function,
281+
* each time an increase has not been requested (!iowait_boost_pending).
282+
*
283+
* A CPU which also appears to have been idle for at least one tick has also
284+
* its IO boost utilization reset.
285+
*
286+
* This mechanism is designed to boost high frequently IO waiting tasks, while
287+
* being more conservative on tasks which does sporadic IO operations.
288+
*/
289+
static void sugov_iowait_apply(struct sugov_cpu *sg_cpu, u64 time,
290+
unsigned long *util, unsigned long *max)
229291
{
230292
unsigned int boost_util, boost_max;
231293

294+
/* No boost currently required */
232295
if (!sg_cpu->iowait_boost)
233296
return;
234297

298+
/* Reset boost if the CPU appears to have been idle enough */
299+
if (sugov_iowait_reset(sg_cpu, time, false))
300+
return;
301+
302+
/*
303+
* An IO waiting task has just woken up:
304+
* allow to further double the boost value
305+
*/
235306
if (sg_cpu->iowait_boost_pending) {
236307
sg_cpu->iowait_boost_pending = false;
237308
} else {
309+
/*
310+
* Otherwise: reduce the boost value and disable it when we
311+
* reach the minimum.
312+
*/
238313
sg_cpu->iowait_boost >>= 1;
239314
if (sg_cpu->iowait_boost < sg_cpu->sg_policy->policy->min) {
240315
sg_cpu->iowait_boost = 0;
241316
return;
242317
}
243318
}
244319

320+
/*
321+
* Apply the current boost value: a CPU is boosted only if its current
322+
* utilization is smaller then the current IO boost level.
323+
*/
245324
boost_util = sg_cpu->iowait_boost;
246325
boost_max = sg_cpu->iowait_boost_max;
247-
248326
if (*util * boost_max < *max * boost_util) {
249327
*util = boost_util;
250328
*max = boost_max;
@@ -283,7 +361,7 @@ static void sugov_update_single(struct update_util_data *hook, u64 time,
283361
unsigned int next_f;
284362
bool busy;
285363

286-
sugov_set_iowait_boost(sg_cpu, time, flags);
364+
sugov_iowait_boost(sg_cpu, time, flags);
287365
sg_cpu->last_update = time;
288366

289367
ignore_dl_rate_limit(sg_cpu, sg_policy);
@@ -296,7 +374,7 @@ static void sugov_update_single(struct update_util_data *hook, u64 time,
296374
sugov_get_util(sg_cpu);
297375
max = sg_cpu->max;
298376
util = sugov_aggregate_util(sg_cpu);
299-
sugov_iowait_boost(sg_cpu, &util, &max);
377+
sugov_iowait_apply(sg_cpu, time, &util, &max);
300378
next_f = get_next_freq(sg_policy, util, max);
301379
/*
302380
* Do not reduce the frequency if the CPU has not been idle
@@ -322,28 +400,12 @@ static unsigned int sugov_next_freq_shared(struct sugov_cpu *sg_cpu, u64 time)
322400
for_each_cpu(j, policy->cpus) {
323401
struct sugov_cpu *j_sg_cpu = &per_cpu(sugov_cpu, j);
324402
unsigned long j_util, j_max;
325-
s64 delta_ns;
326403

327404
sugov_get_util(j_sg_cpu);
328-
329-
/*
330-
* If the CFS CPU utilization was last updated before the
331-
* previous frequency update and the time elapsed between the
332-
* last update of the CPU utilization and the last frequency
333-
* update is long enough, reset iowait_boost and util_cfs, as
334-
* they are now probably stale. However, still consider the
335-
* CPU contribution if it has some DEADLINE utilization
336-
* (util_dl).
337-
*/
338-
delta_ns = time - j_sg_cpu->last_update;
339-
if (delta_ns > TICK_NSEC) {
340-
j_sg_cpu->iowait_boost = 0;
341-
j_sg_cpu->iowait_boost_pending = false;
342-
}
343-
344405
j_max = j_sg_cpu->max;
345406
j_util = sugov_aggregate_util(j_sg_cpu);
346-
sugov_iowait_boost(j_sg_cpu, &j_util, &j_max);
407+
sugov_iowait_apply(j_sg_cpu, time, &j_util, &j_max);
408+
347409
if (j_util * max > j_max * util) {
348410
util = j_util;
349411
max = j_max;
@@ -362,7 +424,7 @@ sugov_update_shared(struct update_util_data *hook, u64 time, unsigned int flags)
362424

363425
raw_spin_lock(&sg_policy->update_lock);
364426

365-
sugov_set_iowait_boost(sg_cpu, time, flags);
427+
sugov_iowait_boost(sg_cpu, time, flags);
366428
sg_cpu->last_update = time;
367429

368430
ignore_dl_rate_limit(sg_cpu, sg_policy);

0 commit comments

Comments
 (0)