[PATCH] Fix idle notifiers

Andi Kleen · Andi Kleen · commit a15da49debaf · 2006-09-26T10:52:40.000+02:00
Previously exit_idle would be called more often than enter_idle

Now instead of using complicated tests just keep track of it
using the per CPU variable as a flip flop.  I moved the idle state into the
PDA to make the access more efficient.

Original bug report and an initial patch from Stephane Eranian,
but redone by AK.

Cc: Stephane Eranian &lt;eranian@hpl.hp.com&gt;

Signed-off-by: Andi Kleen &lt;ak@suse.de&gt;
diff --git a/arch/x86_64/kernel/process.c b/arch/x86_64/kernel/process.c
@@ -80,25 +80,25 @@ void idle_notifier_unregister(struct notifier_block *n)
 }
 EXPORT_SYMBOL(idle_notifier_unregister);
 
-enum idle_state { CPU_IDLE, CPU_NOT_IDLE };
-static DEFINE_PER_CPU(enum idle_state, idle_state) = CPU_NOT_IDLE;
-
 void enter_idle(void)
 {
-	__get_cpu_var(idle_state) = CPU_IDLE;
+	write_pda(isidle, 1);
 	atomic_notifier_call_chain(&idle_notifier, IDLE_START, NULL);
 }
 
 static void __exit_idle(void)
 {
-	__get_cpu_var(idle_state) = CPU_NOT_IDLE;
+	if (read_pda(isidle) == 0)
+		return;
+	write_pda(isidle, 0);
 	atomic_notifier_call_chain(&idle_notifier, IDLE_END, NULL);
 }
 
 /* Called from interrupts to signify idle end */
 void exit_idle(void)
 {
-	if (current->pid | read_pda(irqcount))
+	/* idle loop has pid 0 */
+	if (current->pid)
 		return;
 	__exit_idle();
 }
@@ -220,6 +220,9 @@ void cpu_idle (void)
 				play_dead();
 			enter_idle();
 			idle();
+			/* In many cases the interrupt that ended idle
+			   has already called exit_idle. But some idle
+			   loops can be woken up without interrupt. */
 			__exit_idle();
 		}
 
diff --git a/include/asm-x86_64/pda.h b/include/asm-x86_64/pda.h
@@ -25,7 +25,8 @@ struct x8664_pda {
 	int nodenumber;		    /* number of current node */
 	unsigned int __softirq_pending;
 	unsigned int __nmi_count;	/* number of NMI on this CPUs */
-	int mmu_state;     
+	short mmu_state;
+	short isidle;
 	struct mm_struct *active_mm;
 	unsigned apic_timer_irqs;
 } ____cacheline_aligned_in_smp;

Original file line number	Diff line number	Diff line change
`@@ -80,25 +80,25 @@ void idle_notifier_unregister(struct notifier_block *n)`
`80`	`80`	`}`
`81`	`81`	`EXPORT_SYMBOL(idle_notifier_unregister);`
`82`	`82`
`83`		`-enum idle_state { CPU_IDLE, CPU_NOT_IDLE };`
`84`		`-static DEFINE_PER_CPU(enum idle_state, idle_state) = CPU_NOT_IDLE;`
`85`		`-`
`86`	`83`	`void enter_idle(void)`
`87`	`84`	`{`
`88`		`- __get_cpu_var(idle_state) = CPU_IDLE;`
	`85`	`+ write_pda(isidle, 1);`
`89`	`86`	`atomic_notifier_call_chain(&idle_notifier, IDLE_START, NULL);`
`90`	`87`	`}`
`91`	`88`
`92`	`89`	`static void __exit_idle(void)`
`93`	`90`	`{`
`94`		`- __get_cpu_var(idle_state) = CPU_NOT_IDLE;`
	`91`	`+ if (read_pda(isidle) == 0)`
	`92`	`+ return;`
	`93`	`+ write_pda(isidle, 0);`
`95`	`94`	`atomic_notifier_call_chain(&idle_notifier, IDLE_END, NULL);`
`96`	`95`	`}`
`97`	`96`
`98`	`97`	`/* Called from interrupts to signify idle end */`
`99`	`98`	`void exit_idle(void)`
`100`	`99`	`{`
`101`		`- if (current->pid \| read_pda(irqcount))`
	`100`	`+ /* idle loop has pid 0 */`
	`101`	`+ if (current->pid)`
`102`	`102`	`return;`
`103`	`103`	`__exit_idle();`
`104`	`104`	`}`
`@@ -220,6 +220,9 @@ void cpu_idle (void)`
`220`	`220`	`play_dead();`
`221`	`221`	`enter_idle();`
`222`	`222`	`idle();`
	`223`	`+ /* In many cases the interrupt that ended idle`
	`224`	`+ has already called exit_idle. But some idle`
	`225`	`+ loops can be woken up without interrupt. */`
`223`	`226`	`__exit_idle();`
`224`	`227`	`}`
`225`	`228`