Skip to content

Commit a724632

Browse files
KAGA-KOKOKAGA-KOKO
committed
cpu/hotplug: Rework callback invocation logic
This is preparation for the following patch. This rework here changes the arguments of cpuhp_invoke_callback(). It passes now `state' and whether `startup' or `teardown' callback should be invoked. The callback then is looked up by the function. The following is a clanup of callers: - cpuhp_issue_call() has one argument less - struct cpuhp_cpu_state (which is used by the hotplug thread) gets also its callback removed. The decision if it is a single callback invocation moved to the `single' variable. Also a `bringup' variable has been added to distinguish between startup and teardown callback. - take_cpu_down() needs to start one step earlier. We always get here via CPUHP_TEARDOWN_CPU callback. Before that change cpuhp_ap_states + CPUHP_TEARDOWN_CPU pointed to an empty entry because TEARDOWN is saved in bp_states for this reason. Now that we use cpuhp_get_step() to lookup the state we must explicitly skip it in order not to invoke it twice. Signed-off-by: Thomas Gleixner <tglx@linutronix.de> Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de> Cc: Mark Rutland <mark.rutland@arm.com> Cc: Peter Zijlstra <peterz@infradead.org> Cc: Will Deacon <will.deacon@arm.com> Cc: rt@linutronix.de Link: http://lkml.kernel.org/r/1471024183-12666-2-git-send-email-bigeasy@linutronix.de Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
1 parent 0cb7bf6 commit a724632

File tree

1 file changed

+80
-82
lines changed

1 file changed

+80
-82
lines changed

‎kernel/cpu.c

Lines changed: 80 additions & 82 deletions
Original file line numberDiff line numberDiff line change
@@ -37,8 +37,9 @@
3737
* @thread: Pointer to the hotplug thread
3838
* @should_run: Thread should execute
3939
* @rollback: Perform a rollback
40-
* @cb_stat: The state for a single callback (install/uninstall)
41-
* @cb: Single callback function (install/uninstall)
40+
* @single: Single callback invocation
41+
* @bringup: Single callback bringup or teardown selector
42+
* @cb_state: The state for a single callback (install/uninstall)
4243
* @result: Result of the operation
4344
* @done: Signal completion to the issuer of the task
4445
*/
@@ -49,8 +50,9 @@ struct cpuhp_cpu_state {
4950
struct task_struct *thread;
5051
bool should_run;
5152
bool rollback;
53+
bool single;
54+
bool bringup;
5255
enum cpuhp_state cb_state;
53-
int (*cb)(unsigned int cpu);
5456
int result;
5557
struct completion done;
5658
#endif
@@ -79,24 +81,43 @@ static DEFINE_MUTEX(cpuhp_state_mutex);
7981
static struct cpuhp_step cpuhp_bp_states[];
8082
static struct cpuhp_step cpuhp_ap_states[];
8183

84+
static bool cpuhp_is_ap_state(enum cpuhp_state state)
85+
{
86+
/*
87+
* The extra check for CPUHP_TEARDOWN_CPU is only for documentation
88+
* purposes as that state is handled explicitly in cpu_down.
89+
*/
90+
return state > CPUHP_BRINGUP_CPU && state != CPUHP_TEARDOWN_CPU;
91+
}
92+
93+
static struct cpuhp_step *cpuhp_get_step(enum cpuhp_state state)
94+
{
95+
struct cpuhp_step *sp;
96+
97+
sp = cpuhp_is_ap_state(state) ? cpuhp_ap_states : cpuhp_bp_states;
98+
return sp + state;
99+
}
100+
82101
/**
83102
* cpuhp_invoke_callback _ Invoke the callbacks for a given state
84103
* @cpu: The cpu for which the callback should be invoked
85104
* @step: The step in the state machine
86-
* @cb: The callback function to invoke
105+
* @bringup: True if the bringup callback should be invoked
87106
*
88107
* Called from cpu hotplug and from the state register machinery
89108
*/
90-
static int cpuhp_invoke_callback(unsigned int cpu, enum cpuhp_state step,
91-
int (*cb)(unsigned int))
109+
static int cpuhp_invoke_callback(unsigned int cpu, enum cpuhp_state state,
110+
bool bringup)
92111
{
93112
struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
113+
struct cpuhp_step *step = cpuhp_get_step(state);
114+
int (*cb)(unsigned int cpu) = bringup ? step->startup : step->teardown;
94115
int ret = 0;
95116

96117
if (cb) {
97-
trace_cpuhp_enter(cpu, st->target, step, cb);
118+
trace_cpuhp_enter(cpu, st->target, state, cb);
98119
ret = cb(cpu);
99-
trace_cpuhp_exit(cpu, st->state, step, ret);
120+
trace_cpuhp_exit(cpu, st->state, state, ret);
100121
}
101122
return ret;
102123
}
@@ -371,62 +392,55 @@ static int bringup_cpu(unsigned int cpu)
371392
/*
372393
* Hotplug state machine related functions
373394
*/
374-
static void undo_cpu_down(unsigned int cpu, struct cpuhp_cpu_state *st,
375-
struct cpuhp_step *steps)
395+
static void undo_cpu_down(unsigned int cpu, struct cpuhp_cpu_state *st)
376396
{
377397
for (st->state++; st->state < st->target; st->state++) {
378-
struct cpuhp_step *step = steps + st->state;
398+
struct cpuhp_step *step = cpuhp_get_step(st->state);
379399

380400
if (!step->skip_onerr)
381-
cpuhp_invoke_callback(cpu, st->state, step->startup);
401+
cpuhp_invoke_callback(cpu, st->state, true);
382402
}
383403
}
384404

385405
static int cpuhp_down_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st,
386-
struct cpuhp_step *steps, enum cpuhp_state target)
406+
enum cpuhp_state target)
387407
{
388408
enum cpuhp_state prev_state = st->state;
389409
int ret = 0;
390410

391411
for (; st->state > target; st->state--) {
392-
struct cpuhp_step *step = steps + st->state;
393-
394-
ret = cpuhp_invoke_callback(cpu, st->state, step->teardown);
412+
ret = cpuhp_invoke_callback(cpu, st->state, false);
395413
if (ret) {
396414
st->target = prev_state;
397-
undo_cpu_down(cpu, st, steps);
415+
undo_cpu_down(cpu, st);
398416
break;
399417
}
400418
}
401419
return ret;
402420
}
403421

404-
static void undo_cpu_up(unsigned int cpu, struct cpuhp_cpu_state *st,
405-
struct cpuhp_step *steps)
422+
static void undo_cpu_up(unsigned int cpu, struct cpuhp_cpu_state *st)
406423
{
407424
for (st->state--; st->state > st->target; st->state--) {
408-
struct cpuhp_step *step = steps + st->state;
425+
struct cpuhp_step *step = cpuhp_get_step(st->state);
409426

410427
if (!step->skip_onerr)
411-
cpuhp_invoke_callback(cpu, st->state, step->teardown);
428+
cpuhp_invoke_callback(cpu, st->state, false);
412429
}
413430
}
414431

415432
static int cpuhp_up_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st,
416-
struct cpuhp_step *steps, enum cpuhp_state target)
433+
enum cpuhp_state target)
417434
{
418435
enum cpuhp_state prev_state = st->state;
419436
int ret = 0;
420437

421438
while (st->state < target) {
422-
struct cpuhp_step *step;
423-
424439
st->state++;
425-
step = steps + st->state;
426-
ret = cpuhp_invoke_callback(cpu, st->state, step->startup);
440+
ret = cpuhp_invoke_callback(cpu, st->state, true);
427441
if (ret) {
428442
st->target = prev_state;
429-
undo_cpu_up(cpu, st, steps);
443+
undo_cpu_up(cpu, st);
430444
break;
431445
}
432446
}
@@ -455,13 +469,13 @@ static int cpuhp_ap_offline(unsigned int cpu, struct cpuhp_cpu_state *st)
455469
{
456470
enum cpuhp_state target = max((int)st->target, CPUHP_TEARDOWN_CPU);
457471

458-
return cpuhp_down_callbacks(cpu, st, cpuhp_ap_states, target);
472+
return cpuhp_down_callbacks(cpu, st, target);
459473
}
460474

461475
/* Execute the online startup callbacks. Used to be CPU_ONLINE */
462476
static int cpuhp_ap_online(unsigned int cpu, struct cpuhp_cpu_state *st)
463477
{
464-
return cpuhp_up_callbacks(cpu, st, cpuhp_ap_states, st->target);
478+
return cpuhp_up_callbacks(cpu, st, st->target);
465479
}
466480

467481
/*
@@ -484,18 +498,20 @@ static void cpuhp_thread_fun(unsigned int cpu)
484498
st->should_run = false;
485499

486500
/* Single callback invocation for [un]install ? */
487-
if (st->cb) {
501+
if (st->single) {
488502
if (st->cb_state < CPUHP_AP_ONLINE) {
489503
local_irq_disable();
490-
ret = cpuhp_invoke_callback(cpu, st->cb_state, st->cb);
504+
ret = cpuhp_invoke_callback(cpu, st->cb_state,
505+
st->bringup);
491506
local_irq_enable();
492507
} else {
493-
ret = cpuhp_invoke_callback(cpu, st->cb_state, st->cb);
508+
ret = cpuhp_invoke_callback(cpu, st->cb_state,
509+
st->bringup);
494510
}
495511
} else if (st->rollback) {
496512
BUG_ON(st->state < CPUHP_AP_ONLINE_IDLE);
497513

498-
undo_cpu_down(cpu, st, cpuhp_ap_states);
514+
undo_cpu_down(cpu, st);
499515
/*
500516
* This is a momentary workaround to keep the notifier users
501517
* happy. Will go away once we got rid of the notifiers.
@@ -517,8 +533,8 @@ static void cpuhp_thread_fun(unsigned int cpu)
517533
}
518534

519535
/* Invoke a single callback on a remote cpu */
520-
static int cpuhp_invoke_ap_callback(int cpu, enum cpuhp_state state,
521-
int (*cb)(unsigned int))
536+
static int
537+
cpuhp_invoke_ap_callback(int cpu, enum cpuhp_state state, bool bringup)
522538
{
523539
struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
524540

@@ -530,10 +546,12 @@ static int cpuhp_invoke_ap_callback(int cpu, enum cpuhp_state state,
530546
* we invoke the thread function directly.
531547
*/
532548
if (!st->thread)
533-
return cpuhp_invoke_callback(cpu, state, cb);
549+
return cpuhp_invoke_callback(cpu, state, bringup);
534550

535551
st->cb_state = state;
536-
st->cb = cb;
552+
st->single = true;
553+
st->bringup = bringup;
554+
537555
/*
538556
* Make sure the above stores are visible before should_run becomes
539557
* true. Paired with the mb() above in cpuhp_thread_fun()
@@ -549,7 +567,7 @@ static int cpuhp_invoke_ap_callback(int cpu, enum cpuhp_state state,
549567
static void __cpuhp_kick_ap_work(struct cpuhp_cpu_state *st)
550568
{
551569
st->result = 0;
552-
st->cb = NULL;
570+
st->single = false;
553571
/*
554572
* Make sure the above stores are visible before should_run becomes
555573
* true. Paired with the mb() above in cpuhp_thread_fun()
@@ -700,12 +718,16 @@ static int take_cpu_down(void *_param)
700718
if (err < 0)
701719
return err;
702720

721+
/*
722+
* We get here while we are in CPUHP_TEARDOWN_CPU state and we must not
723+
* do this step again.
724+
*/
725+
WARN_ON(st->state != CPUHP_TEARDOWN_CPU);
726+
st->state--;
703727
/* Invoke the former CPU_DYING callbacks */
704-
for (; st->state > target; st->state--) {
705-
struct cpuhp_step *step = cpuhp_ap_states + st->state;
728+
for (; st->state > target; st->state--)
729+
cpuhp_invoke_callback(cpu, st->state, false);
706730

707-
cpuhp_invoke_callback(cpu, st->state, step->teardown);
708-
}
709731
/* Give up timekeeping duties */
710732
tick_handover_do_timer();
711733
/* Park the stopper thread */
@@ -844,7 +866,7 @@ static int __ref _cpu_down(unsigned int cpu, int tasks_frozen,
844866
* The AP brought itself down to CPUHP_TEARDOWN_CPU. So we need
845867
* to do the further cleanups.
846868
*/
847-
ret = cpuhp_down_callbacks(cpu, st, cpuhp_bp_states, target);
869+
ret = cpuhp_down_callbacks(cpu, st, target);
848870
if (ret && st->state > CPUHP_TEARDOWN_CPU && st->state < prev_state) {
849871
st->target = prev_state;
850872
st->rollback = true;
@@ -898,11 +920,8 @@ void notify_cpu_starting(unsigned int cpu)
898920
enum cpuhp_state target = min((int)st->target, CPUHP_AP_ONLINE);
899921

900922
while (st->state < target) {
901-
struct cpuhp_step *step;
902-
903923
st->state++;
904-
step = cpuhp_ap_states + st->state;
905-
cpuhp_invoke_callback(cpu, st->state, step->startup);
924+
cpuhp_invoke_callback(cpu, st->state, true);
906925
}
907926
}
908927

@@ -987,7 +1006,7 @@ static int _cpu_up(unsigned int cpu, int tasks_frozen, enum cpuhp_state target)
9871006
* responsible for bringing it up to the target state.
9881007
*/
9891008
target = min((int)target, CPUHP_BRINGUP_CPU);
990-
ret = cpuhp_up_callbacks(cpu, st, cpuhp_bp_states, target);
1009+
ret = cpuhp_up_callbacks(cpu, st, target);
9911010
out:
9921011
cpu_hotplug_done();
9931012
return ret;
@@ -1364,23 +1383,6 @@ static int cpuhp_cb_check(enum cpuhp_state state)
13641383
return 0;
13651384
}
13661385

1367-
static bool cpuhp_is_ap_state(enum cpuhp_state state)
1368-
{
1369-
/*
1370-
* The extra check for CPUHP_TEARDOWN_CPU is only for documentation
1371-
* purposes as that state is handled explicitely in cpu_down.
1372-
*/
1373-
return state > CPUHP_BRINGUP_CPU && state != CPUHP_TEARDOWN_CPU;
1374-
}
1375-
1376-
static struct cpuhp_step *cpuhp_get_step(enum cpuhp_state state)
1377-
{
1378-
struct cpuhp_step *sp;
1379-
1380-
sp = cpuhp_is_ap_state(state) ? cpuhp_ap_states : cpuhp_bp_states;
1381-
return sp + state;
1382-
}
1383-
13841386
static void cpuhp_store_callbacks(enum cpuhp_state state,
13851387
const char *name,
13861388
int (*startup)(unsigned int cpu),
@@ -1406,24 +1408,24 @@ static void *cpuhp_get_teardown_cb(enum cpuhp_state state)
14061408
* Call the startup/teardown function for a step either on the AP or
14071409
* on the current CPU.
14081410
*/
1409-
static int cpuhp_issue_call(int cpu, enum cpuhp_state state,
1410-
int (*cb)(unsigned int), bool bringup)
1411+
static int cpuhp_issue_call(int cpu, enum cpuhp_state state, bool bringup)
14111412
{
1413+
struct cpuhp_step *sp = cpuhp_get_step(state);
14121414
int ret;
14131415

1414-
if (!cb)
1416+
if ((bringup && !sp->startup) || (!bringup && !sp->teardown))
14151417
return 0;
14161418
/*
14171419
* The non AP bound callbacks can fail on bringup. On teardown
14181420
* e.g. module removal we crash for now.
14191421
*/
14201422
#ifdef CONFIG_SMP
14211423
if (cpuhp_is_ap_state(state))
1422-
ret = cpuhp_invoke_ap_callback(cpu, state, cb);
1424+
ret = cpuhp_invoke_ap_callback(cpu, state, bringup);
14231425
else
1424-
ret = cpuhp_invoke_callback(cpu, state, cb);
1426+
ret = cpuhp_invoke_callback(cpu, state, bringup);
14251427
#else
1426-
ret = cpuhp_invoke_callback(cpu, state, cb);
1428+
ret = cpuhp_invoke_callback(cpu, state, bringup);
14271429
#endif
14281430
BUG_ON(ret && !bringup);
14291431
return ret;
@@ -1434,14 +1436,10 @@ static int cpuhp_issue_call(int cpu, enum cpuhp_state state,
14341436
*
14351437
* Note: The teardown callbacks for rollback are not allowed to fail!
14361438
*/
1437-
static void cpuhp_rollback_install(int failedcpu, enum cpuhp_state state,
1438-
int (*teardown)(unsigned int cpu))
1439+
static void cpuhp_rollback_install(int failedcpu, enum cpuhp_state state)
14391440
{
14401441
int cpu;
14411442

1442-
if (!teardown)
1443-
return;
1444-
14451443
/* Roll back the already executed steps on the other cpus */
14461444
for_each_present_cpu(cpu) {
14471445
struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
@@ -1452,7 +1450,7 @@ static void cpuhp_rollback_install(int failedcpu, enum cpuhp_state state,
14521450

14531451
/* Did we invoke the startup call on that cpu ? */
14541452
if (cpustate >= state)
1455-
cpuhp_issue_call(cpu, state, teardown, false);
1453+
cpuhp_issue_call(cpu, state, false);
14561454
}
14571455
}
14581456

@@ -1527,9 +1525,10 @@ int __cpuhp_setup_state(enum cpuhp_state state,
15271525
if (cpustate < state)
15281526
continue;
15291527

1530-
ret = cpuhp_issue_call(cpu, state, startup, true);
1528+
ret = cpuhp_issue_call(cpu, state, true);
15311529
if (ret) {
1532-
cpuhp_rollback_install(cpu, state, teardown);
1530+
if (teardown)
1531+
cpuhp_rollback_install(cpu, state);
15331532
cpuhp_store_callbacks(state, NULL, NULL, NULL);
15341533
goto out;
15351534
}
@@ -1553,14 +1552,13 @@ EXPORT_SYMBOL(__cpuhp_setup_state);
15531552
*/
15541553
void __cpuhp_remove_state(enum cpuhp_state state, bool invoke)
15551554
{
1556-
int (*teardown)(unsigned int cpu) = cpuhp_get_teardown_cb(state);
15571555
int cpu;
15581556

15591557
BUG_ON(cpuhp_cb_check(state));
15601558

15611559
get_online_cpus();
15621560

1563-
if (!invoke || !teardown)
1561+
if (!invoke || !cpuhp_get_teardown_cb(state))
15641562
goto remove;
15651563

15661564
/*
@@ -1573,7 +1571,7 @@ void __cpuhp_remove_state(enum cpuhp_state state, bool invoke)
15731571
int cpustate = st->state;
15741572

15751573
if (cpustate >= state)
1576-
cpuhp_issue_call(cpu, state, teardown, false);
1574+
cpuhp_issue_call(cpu, state, false);
15771575
}
15781576
remove:
15791577
cpuhp_store_callbacks(state, NULL, NULL, NULL);

0 commit comments

Comments
 (0)