Commit bde6c3aa authored by Paul E. McKenney's avatar Paul E. McKenney

rcu: Provide cond_resched_rcu_qs() to force quiescent states in long loops

RCU-tasks requires the occasional voluntary context switch
from CPU-bound in-kernel tasks.  In some cases, this requires
instrumenting cond_resched().  However, there is some reluctance
to countenance unconditionally instrumenting cond_resched() (see
http://lwn.net/Articles/603252/), so this commit creates a separate
cond_resched_rcu_qs() that may be used in place of cond_resched() in
locations prone to long-duration in-kernel looping.

This commit currently instruments only RCU-tasks.  Future possibilities
include also instrumenting RCU, RCU-bh, and RCU-sched in order to reduce
IPI usage.
Signed-off-by: 's avatarPaul E. McKenney <paulmck@linux.vnet.ibm.com>
parent 8315f422
...@@ -367,7 +367,7 @@ static struct fdtable *close_files(struct files_struct * files) ...@@ -367,7 +367,7 @@ static struct fdtable *close_files(struct files_struct * files)
struct file * file = xchg(&fdt->fd[i], NULL); struct file * file = xchg(&fdt->fd[i], NULL);
if (file) { if (file) {
filp_close(file, files); filp_close(file, files);
cond_resched(); cond_resched_rcu_qs();
} }
} }
i++; i++;
......
...@@ -330,6 +330,19 @@ static inline void rcu_user_hooks_switch(struct task_struct *prev, ...@@ -330,6 +330,19 @@ static inline void rcu_user_hooks_switch(struct task_struct *prev,
#define rcu_note_voluntary_context_switch(t) do { } while (0) #define rcu_note_voluntary_context_switch(t) do { } while (0)
#endif /* #else #ifdef CONFIG_TASKS_RCU */ #endif /* #else #ifdef CONFIG_TASKS_RCU */
/**
* cond_resched_rcu_qs - Report potential quiescent states to RCU
*
* This macro resembles cond_resched(), except that it is defined to
* report potential quiescent states to RCU-tasks even if the cond_resched()
* machinery were to be shut off, as some advocate for PREEMPT kernels.
*/
#define cond_resched_rcu_qs() \
do { \
rcu_note_voluntary_context_switch(current); \
cond_resched(); \
} while (0)
#if defined(CONFIG_DEBUG_LOCK_ALLOC) || defined(CONFIG_RCU_TRACE) || defined(CONFIG_SMP) #if defined(CONFIG_DEBUG_LOCK_ALLOC) || defined(CONFIG_RCU_TRACE) || defined(CONFIG_SMP)
bool __rcu_is_watching(void); bool __rcu_is_watching(void);
#endif /* #if defined(CONFIG_DEBUG_LOCK_ALLOC) || defined(CONFIG_RCU_TRACE) || defined(CONFIG_SMP) */ #endif /* #if defined(CONFIG_DEBUG_LOCK_ALLOC) || defined(CONFIG_RCU_TRACE) || defined(CONFIG_SMP) */
......
...@@ -667,7 +667,7 @@ static int rcu_torture_boost(void *arg) ...@@ -667,7 +667,7 @@ static int rcu_torture_boost(void *arg)
} }
call_rcu_time = jiffies; call_rcu_time = jiffies;
} }
cond_resched(); cond_resched_rcu_qs();
stutter_wait("rcu_torture_boost"); stutter_wait("rcu_torture_boost");
if (torture_must_stop()) if (torture_must_stop())
goto checkwait; goto checkwait;
...@@ -1019,7 +1019,7 @@ rcu_torture_reader(void *arg) ...@@ -1019,7 +1019,7 @@ rcu_torture_reader(void *arg)
__this_cpu_inc(rcu_torture_batch[completed]); __this_cpu_inc(rcu_torture_batch[completed]);
preempt_enable(); preempt_enable();
cur_ops->readunlock(idx); cur_ops->readunlock(idx);
cond_resched(); cond_resched_rcu_qs();
stutter_wait("rcu_torture_reader"); stutter_wait("rcu_torture_reader");
} while (!torture_must_stop()); } while (!torture_must_stop());
if (irqreader && cur_ops->irq_capable) { if (irqreader && cur_ops->irq_capable) {
......
...@@ -1647,7 +1647,7 @@ static int rcu_gp_init(struct rcu_state *rsp) ...@@ -1647,7 +1647,7 @@ static int rcu_gp_init(struct rcu_state *rsp)
rnp->level, rnp->grplo, rnp->level, rnp->grplo,
rnp->grphi, rnp->qsmask); rnp->grphi, rnp->qsmask);
raw_spin_unlock_irq(&rnp->lock); raw_spin_unlock_irq(&rnp->lock);
cond_resched(); cond_resched_rcu_qs();
} }
mutex_unlock(&rsp->onoff_mutex); mutex_unlock(&rsp->onoff_mutex);
...@@ -1736,7 +1736,7 @@ static void rcu_gp_cleanup(struct rcu_state *rsp) ...@@ -1736,7 +1736,7 @@ static void rcu_gp_cleanup(struct rcu_state *rsp)
/* smp_mb() provided by prior unlock-lock pair. */ /* smp_mb() provided by prior unlock-lock pair. */
nocb += rcu_future_gp_cleanup(rsp, rnp); nocb += rcu_future_gp_cleanup(rsp, rnp);
raw_spin_unlock_irq(&rnp->lock); raw_spin_unlock_irq(&rnp->lock);
cond_resched(); cond_resched_rcu_qs();
} }
rnp = rcu_get_root(rsp); rnp = rcu_get_root(rsp);
raw_spin_lock_irq(&rnp->lock); raw_spin_lock_irq(&rnp->lock);
...@@ -1785,7 +1785,7 @@ static int __noreturn rcu_gp_kthread(void *arg) ...@@ -1785,7 +1785,7 @@ static int __noreturn rcu_gp_kthread(void *arg)
/* Locking provides needed memory barrier. */ /* Locking provides needed memory barrier. */
if (rcu_gp_init(rsp)) if (rcu_gp_init(rsp))
break; break;
cond_resched(); cond_resched_rcu_qs();
flush_signals(current); flush_signals(current);
trace_rcu_grace_period(rsp->name, trace_rcu_grace_period(rsp->name,
ACCESS_ONCE(rsp->gpnum), ACCESS_ONCE(rsp->gpnum),
...@@ -1828,10 +1828,10 @@ static int __noreturn rcu_gp_kthread(void *arg) ...@@ -1828,10 +1828,10 @@ static int __noreturn rcu_gp_kthread(void *arg)
trace_rcu_grace_period(rsp->name, trace_rcu_grace_period(rsp->name,
ACCESS_ONCE(rsp->gpnum), ACCESS_ONCE(rsp->gpnum),
TPS("fqsend")); TPS("fqsend"));
cond_resched(); cond_resched_rcu_qs();
} else { } else {
/* Deal with stray signal. */ /* Deal with stray signal. */
cond_resched(); cond_resched_rcu_qs();
flush_signals(current); flush_signals(current);
trace_rcu_grace_period(rsp->name, trace_rcu_grace_period(rsp->name,
ACCESS_ONCE(rsp->gpnum), ACCESS_ONCE(rsp->gpnum),
...@@ -2434,7 +2434,7 @@ static void force_qs_rnp(struct rcu_state *rsp, ...@@ -2434,7 +2434,7 @@ static void force_qs_rnp(struct rcu_state *rsp,
struct rcu_node *rnp; struct rcu_node *rnp;
rcu_for_each_leaf_node(rsp, rnp) { rcu_for_each_leaf_node(rsp, rnp) {
cond_resched(); cond_resched_rcu_qs();
mask = 0; mask = 0;
raw_spin_lock_irqsave(&rnp->lock, flags); raw_spin_lock_irqsave(&rnp->lock, flags);
smp_mb__after_unlock_lock(); smp_mb__after_unlock_lock();
......
...@@ -1848,7 +1848,7 @@ static int rcu_oom_notify(struct notifier_block *self, ...@@ -1848,7 +1848,7 @@ static int rcu_oom_notify(struct notifier_block *self,
get_online_cpus(); get_online_cpus();
for_each_online_cpu(cpu) { for_each_online_cpu(cpu) {
smp_call_function_single(cpu, rcu_oom_notify_cpu, NULL, 1); smp_call_function_single(cpu, rcu_oom_notify_cpu, NULL, 1);
cond_resched(); cond_resched_rcu_qs();
} }
put_online_cpus(); put_online_cpus();
......
...@@ -789,7 +789,7 @@ static int do_mlockall(int flags) ...@@ -789,7 +789,7 @@ static int do_mlockall(int flags)
/* Ignore errors */ /* Ignore errors */
mlock_fixup(vma, &prev, vma->vm_start, vma->vm_end, newflags); mlock_fixup(vma, &prev, vma->vm_start, vma->vm_end, newflags);
cond_resched(); cond_resched_rcu_qs();
} }
out: out:
return 0; return 0;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment