[PATCH] softirq: Prefer low-latency over ksoftirqd for HI_SOFTIRQ

[Chris Wilson]

References: 4cd13c21b207 ("softirq: Let ksoftirqd do its job")
Signed-off-by: Chris Wilson <chris at chris-wilson.co.uk>
---
 kernel/softirq.c | 23 +++++++++++++++--------
 1 file changed, 15 insertions(+), 8 deletions(-)

diff --git a/kernel/softirq.c b/kernel/softirq.c
index 24d243ef8e71..c18da7bfff08 100644
--- a/kernel/softirq.c
+++ b/kernel/softirq.c
@@ -81,11 +81,15 @@ static void wakeup_softirqd(void)
  * If ksoftirqd is scheduled, we do not want to process pending softirqs
  * right now. Let ksoftirqd handle this at its own rate, to get fairness.
  */
-static bool ksoftirqd_running(void)
+static bool ksoftirqd_running(__u32 pending)
 {
-	struct task_struct *tsk = __this_cpu_read(ksoftirqd);
+	struct task_struct *tsk;
+
+	if (pending & HI_SOFTIRQ)
+		return false;
 
-	return tsk && (tsk->state == TASK_RUNNING);
+	tsk = __this_cpu_read(ksoftirqd);
+	return tsk && !(tsk->state & TASK_NORMAL);
 }
 
 /*
@@ -325,7 +329,7 @@ asmlinkage __visible void do_softirq(void)
 
 	pending = local_softirq_pending();
 
-	if (pending && !ksoftirqd_running())
+	if (pending && !ksoftirqd_running(pending))
 		do_softirq_own_stack();
 
 	local_irq_restore(flags);
@@ -350,12 +354,15 @@ void irq_enter(void)
 	__irq_enter();
 }
 
-static inline void invoke_softirq(void)
+static inline void invoke_softirq(__u32 pending)
 {
-	if (ksoftirqd_running())
+	if (!pending)
 		return;
 
 	if (!force_irqthreads) {
+		if (ksoftirqd_running(pending))
+			return;
+
 #ifdef CONFIG_HAVE_IRQ_EXIT_ON_IRQ_STACK
 		/*
 		 * We can safely execute softirq on the current stack if
@@ -401,8 +408,8 @@ void irq_exit(void)
 #endif
 	account_irq_exit_time(current);
 	preempt_count_sub(HARDIRQ_OFFSET);
-	if (!in_interrupt() && local_softirq_pending())
-		invoke_softirq();
+	if (!in_interrupt())
+		invoke_softirq(local_softirq_pending());
 
 	tick_irq_exit();
 	rcu_irq_exit();
-- 
2.16.2