diff mbox

[tip/core/rcu,07/86] rcu: Force per-rcu_node kthreads off of the outgoing CPU

Message ID 1304256126-26015-7-git-send-email-paulmck@linux.vnet.ibm.com
State New
Headers show

Commit Message

Paul E. McKenney May 1, 2011, 1:20 p.m. UTC 
From: Paul E. McKenney <paul.mckenney@linaro.org>

The scheduler has had some heartburn in the past when too many real-time
kthreads were affinitied to the outgoing CPU.  So, this commit lightens
the load by forcing the per-rcu_node and the boost kthreads off of the
outgoing CPU.  Note that RCU's per-CPU kthread remains on the outgoing
CPU until the bitter end, as it must in order to preserve correctness.

Also avoid disabling hardirqs across calls to set_cpus_allowed_ptr(),
given that this function can block.

Signed-off-by: Paul E. McKenney <paul.mckenney@linaro.org>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
---
 kernel/rcutree.c        |   24 +++++++++++++++++++-----
 kernel/rcutree_plugin.h |    8 +++++---
 2 files changed, 24 insertions(+), 8 deletions(-)

Comments

Josh Triplett May 1, 2011, 3:10 p.m. UTC | #1 
On Sun, May 01, 2011 at 06:20:47AM -0700, Paul E. McKenney wrote:
> From: Paul E. McKenney <paul.mckenney@linaro.org>
> 
> The scheduler has had some heartburn in the past when too many real-time
> kthreads were affinitied to the outgoing CPU.  So, this commit lightens
> the load by forcing the per-rcu_node and the boost kthreads off of the
> outgoing CPU.  Note that RCU's per-CPU kthread remains on the outgoing
> CPU until the bitter end, as it must in order to preserve correctness.
> 
> Also avoid disabling hardirqs across calls to set_cpus_allowed_ptr(),
> given that this function can block.

This seems like two separate changes in one patch.  The change to
kernel/rcutree_plugin.h seems entirely separable from the change to
kernel/rcutree.c here.

> Signed-off-by: Paul E. McKenney <paul.mckenney@linaro.org>
> Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>

Heh, amusingly this actually seems meaningful, if you intend it as a
"role" approval.

- Josh Triplett
Paul E. McKenney May 2, 2011, 10:25 a.m. UTC | #2 
On Sun, May 01, 2011 at 08:10:35AM -0700, Josh Triplett wrote:
> On Sun, May 01, 2011 at 06:20:47AM -0700, Paul E. McKenney wrote:
> > From: Paul E. McKenney <paul.mckenney@linaro.org>
> > 
> > The scheduler has had some heartburn in the past when too many real-time
> > kthreads were affinitied to the outgoing CPU.  So, this commit lightens
> > the load by forcing the per-rcu_node and the boost kthreads off of the
> > outgoing CPU.  Note that RCU's per-CPU kthread remains on the outgoing
> > CPU until the bitter end, as it must in order to preserve correctness.
> > 
> > Also avoid disabling hardirqs across calls to set_cpus_allowed_ptr(),
> > given that this function can block.
> 
> This seems like two separate changes in one patch.  The change to
> kernel/rcutree_plugin.h seems entirely separable from the change to
> kernel/rcutree.c here.
> 
> > Signed-off-by: Paul E. McKenney <paul.mckenney@linaro.org>
> > Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
> 
> Heh, amusingly this actually seems meaningful, if you intend it as a
> "role" approval.

Yep, that is the intent!  ;-)

							Thanx, Paul
diff mbox

Patch

diff --git a/kernel/rcutree.c b/kernel/rcutree.c
index 1ce51b8..85b06c5 100644
--- a/kernel/rcutree.c
+++ b/kernel/rcutree.c
@@ -95,7 +95,7 @@  static DEFINE_PER_CPU(wait_queue_head_t, rcu_cpu_wq);
 static DEFINE_PER_CPU(char, rcu_cpu_has_work);
 static char rcu_kthreads_spawnable;
 
-static void rcu_node_kthread_setaffinity(struct rcu_node *rnp);
+static void rcu_node_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu);
 static void invoke_rcu_cpu_kthread(void);
 
 #define RCU_KTHREAD_PRIO 1	/* RT priority for per-CPU kthreads. */
@@ -1099,7 +1099,7 @@  static void __rcu_offline_cpu(int cpu, struct rcu_state *rsp)
 		kthread_stop(t);
 		rcu_stop_boost_kthread(rnp);
 	} else
-		rcu_node_kthread_setaffinity(rnp);
+		rcu_node_kthread_setaffinity(rnp, -1);
 }
 
 /*
@@ -1644,8 +1644,12 @@  static int rcu_node_kthread(void *arg)
  * Set the per-rcu_node kthread's affinity to cover all CPUs that are
  * served by the rcu_node in question.  The CPU hotplug lock is still
  * held, so the value of rnp->qsmaskinit will be stable.
+ *
+ * We don't include outgoingcpu in the affinity set, use -1 if there is
+ * no outgoing CPU.  If there are no CPUs left in the affinity set,
+ * this function allows the kthread to execute on any CPU.
  */
-static void rcu_node_kthread_setaffinity(struct rcu_node *rnp)
+static void rcu_node_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu)
 {
 	cpumask_var_t cm;
 	int cpu;
@@ -1657,8 +1661,14 @@  static void rcu_node_kthread_setaffinity(struct rcu_node *rnp)
 		return;
 	cpumask_clear(cm);
 	for (cpu = rnp->grplo; cpu <= rnp->grphi; cpu++, mask >>= 1)
-		if (mask & 0x1)
+		if ((mask & 0x1) && cpu != outgoingcpu)
 			cpumask_set_cpu(cpu, cm);
+	if (cpumask_weight(cm) == 0) {
+		cpumask_setall(cm);
+		for (cpu = rnp->grplo; cpu <= rnp->grphi; cpu++)
+			cpumask_clear_cpu(cpu, cm);
+		WARN_ON_ONCE(cpumask_weight(cm) == 0);
+	}
 	set_cpus_allowed_ptr(rnp->node_kthread_task, cm);
 	rcu_boost_kthread_setaffinity(rnp, cm);
 	free_cpumask_var(cm);
@@ -2154,7 +2164,11 @@  static int __cpuinit rcu_cpu_notify(struct notifier_block *self,
 		rcu_online_kthreads(cpu);
 		break;
 	case CPU_ONLINE:
-		rcu_node_kthread_setaffinity(rnp);
+	case CPU_DOWN_FAILED:
+		rcu_node_kthread_setaffinity(rnp, -1);
+		break;
+	case CPU_DOWN_PREPARE:
+		rcu_node_kthread_setaffinity(rnp, cpu);
 		break;
 	case CPU_DYING:
 	case CPU_DYING_FROZEN:
diff --git a/kernel/rcutree_plugin.h b/kernel/rcutree_plugin.h
index 5964f82..4e48625 100644
--- a/kernel/rcutree_plugin.h
+++ b/kernel/rcutree_plugin.h
@@ -1212,17 +1212,19 @@  static void rcu_initiate_boost(struct rcu_node *rnp)
 	}
 }
 
+/*
+ * Set the affinity of the boost kthread.  The CPU-hotplug locks are
+ * held, so no one should be messing with the existence of the boost
+ * kthread.
+ */
 static void rcu_boost_kthread_setaffinity(struct rcu_node *rnp,
 					  cpumask_var_t cm)
 {
-	unsigned long flags;
 	struct task_struct *t;
 
-	raw_spin_lock_irqsave(&rnp->lock, flags);
 	t = rnp->boost_kthread_task;
 	if (t != NULL)
 		set_cpus_allowed_ptr(rnp->boost_kthread_task, cm);
-	raw_spin_unlock_irqrestore(&rnp->lock, flags);
 }
 
 #define RCU_BOOST_DELAY_JIFFIES DIV_ROUND_UP(CONFIG_RCU_BOOST_DELAY * HZ, 1000)