@@ -1031,14 +1031,15 @@ explains which is which.
CPU#: The CPU which the process was running on.
irqs-off: 'd' interrupts are disabled. '.' otherwise.
- .. caution:: If the architecture does not support a way to
- read the irq flags variable, an 'X' will always
- be printed here.
need-resched:
+ - 'B' all, TIF_NEED_RESCHED, PREEMPT_NEED_RESCHED and TIF_RESCHED_LAZY is set,
- 'N' both TIF_NEED_RESCHED and PREEMPT_NEED_RESCHED is set,
- 'n' only TIF_NEED_RESCHED is set,
- 'p' only PREEMPT_NEED_RESCHED is set,
+ - 'L' borg PREEMPT_NEED_RESCHED and TIF_RESCHED_LAZY is set,
+ - 'b' borg TIF_NEED_RESCHED and TIF_RESCHED_LAZY is set,
+ - 'l' only TIF_RESCHED_LAZY is set
- '.' otherwise.
hardirq/softirq:
@@ -616,14 +616,27 @@ extern void __raise_softirq_irqoff(unsigned int nr);
extern void raise_softirq_irqoff(unsigned int nr);
extern void raise_softirq(unsigned int nr);
+/*
+ * Handle timers in a dedicated thread at a low SCHED_FIFO priority instead in
+ * ksoftirqd as to be prefred over SCHED_NORMAL tasks.
+ */
#ifdef CONFIG_PREEMPT_RT
DECLARE_PER_CPU(struct task_struct *, timersd);
DECLARE_PER_CPU(unsigned long, pending_timer_softirq);
-extern void raise_timer_softirq(void);
-extern void raise_hrtimer_softirq(void);
+void raise_ktimers_thread(unsigned int nr);
-static inline unsigned int local_pending_timers(void)
+static inline void raise_timer_softirq(void)
+{
+ raise_ktimers_thread(TIMER_SOFTIRQ);
+}
+
+static inline void raise_hrtimer_softirq(void)
+{
+ raise_ktimers_thread(HRTIMER_SOFTIRQ);
+}
+
+static inline unsigned int local_timers_pending(void)
{
return __this_cpu_read(pending_timer_softirq);
}
@@ -631,15 +644,17 @@ static inline unsigned int local_pending_timers(void)
#else
static inline void raise_timer_softirq(void)
{
- raise_softirq(TIMER_SOFTIRQ);
+ lockdep_assert_in_irq();
+ __raise_softirq_irqoff(TIMER_SOFTIRQ);
}
static inline void raise_hrtimer_softirq(void)
{
- raise_softirq_irqoff(HRTIMER_SOFTIRQ);
+ lockdep_assert_in_irq();
+ __raise_softirq_irqoff(HRTIMER_SOFTIRQ);
}
-static inline unsigned int local_pending_timers(void)
+static inline unsigned int local_timers_pending(void)
{
return local_softirq_pending();
}
@@ -184,8 +184,8 @@ unsigned int tracing_gen_ctx_irq_test(unsigned int irqs_status);
enum trace_flag_type {
TRACE_FLAG_IRQS_OFF = 0x01,
- TRACE_FLAG_NEED_RESCHED = 0x02,
- TRACE_FLAG_NEED_RESCHED_LAZY = 0x04,
+ TRACE_FLAG_NEED_RESCHED_LAZY = 0x02,
+ TRACE_FLAG_NEED_RESCHED = 0x04,
TRACE_FLAG_HARDIRQ = 0x08,
TRACE_FLAG_SOFTIRQ = 0x10,
TRACE_FLAG_PREEMPT_RESCHED = 0x20,
@@ -193,7 +193,6 @@ enum trace_flag_type {
TRACE_FLAG_BH_OFF = 0x80,
};
-#ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
static inline unsigned int tracing_gen_ctx_flags(unsigned long irqflags)
{
unsigned int irq_status = irqs_disabled_flags(irqflags) ?
@@ -207,17 +206,6 @@ static inline unsigned int tracing_gen_ctx(void)
local_save_flags(irqflags);
return tracing_gen_ctx_flags(irqflags);
}
-#else
-
-static inline unsigned int tracing_gen_ctx_flags(unsigned long irqflags)
-{
- return tracing_gen_ctx_irq_test(0);
-}
-static inline unsigned int tracing_gen_ctx(void)
-{
- return tracing_gen_ctx_irq_test(0);
-}
-#endif
static inline unsigned int tracing_gen_ctx_dec(void)
{
@@ -1125,11 +1125,11 @@ static __always_inline bool dynamic_preempt_promote(void)
#else
static __always_inline bool dynamic_preempt_lazy(void)
{
- return IS_ENABLED(PREEMPT_LAZY) | IS_ENABLED(PREEMPT_LAZIEST);
+ return IS_ENABLED(CONFIG_PREEMPT_LAZY) | IS_ENABLED(CONFIG_PREEMPT_LAZIEST);
}
static __always_inline bool dynamic_preempt_promote(void)
{
- return IS_ENABLED(PREEMPT_LAZY);
+ return IS_ENABLED(CONFIG_PREEMPT_LAZY);
}
#endif
@@ -654,7 +654,7 @@ static inline void __irq_exit_rcu(void)
if (!in_interrupt() && local_softirq_pending())
invoke_softirq();
- if (IS_ENABLED(CONFIG_PREEMPT_RT) && local_pending_timers() &&
+ if (IS_ENABLED(CONFIG_PREEMPT_RT) && local_timers_pending() &&
!(in_nmi() | in_hardirq()))
wake_timersd();
@@ -996,12 +996,20 @@ static struct smp_hotplug_thread softirq_threads = {
#ifdef CONFIG_PREEMPT_RT
static void timersd_setup(unsigned int cpu)
{
+ /* Above SCHED_NORMAL to handle timers before regular tasks. */
sched_set_fifo_low(current);
}
static int timersd_should_run(unsigned int cpu)
{
- return local_pending_timers();
+ return local_timers_pending();
+}
+
+void raise_ktimers_thread(unsigned int nr)
+{
+ lockdep_assert_in_irq();
+ trace_softirq_raise(nr);
+ __this_cpu_or(pending_timer_softirq, 1 << nr);
}
static void run_timersd(unsigned int cpu)
@@ -1010,7 +1018,7 @@ static void run_timersd(unsigned int cpu)
ksoftirqd_run_begin();
- timer_si = local_pending_timers();
+ timer_si = local_timers_pending();
__this_cpu_write(pending_timer_softirq, 0);
or_softirq_pending(timer_si);
@@ -1019,27 +1027,6 @@ static void run_timersd(unsigned int cpu)
ksoftirqd_run_end();
}
-static void raise_ktimers_thread(unsigned int nr)
-{
- trace_softirq_raise(nr);
- __this_cpu_or(pending_timer_softirq, 1 << nr);
-}
-
-void raise_hrtimer_softirq(void)
-{
- raise_ktimers_thread(HRTIMER_SOFTIRQ);
-}
-
-void raise_timer_softirq(void)
-{
- unsigned long flags;
-
- local_irq_save(flags);
- raise_ktimers_thread(TIMER_SOFTIRQ);
- wake_timersd();
- local_irq_restore(flags);
-}
-
static struct smp_hotplug_thread timer_threads = {
.store = &timersd,
.setup = timersd_setup,
@@ -865,7 +865,7 @@ static void tick_nohz_restart(struct tick_sched *ts, ktime_t now)
static inline bool local_timer_softirq_pending(void)
{
- return local_pending_timers() & BIT(TIMER_SOFTIRQ);
+ return local_timers_pending() & BIT(TIMER_SOFTIRQ);
}
/*
@@ -460,7 +460,6 @@ int trace_print_lat_fmt(struct trace_seq *s, struct trace_entry *entry)
(entry->flags & TRACE_FLAG_IRQS_OFF && bh_off) ? 'D' :
(entry->flags & TRACE_FLAG_IRQS_OFF) ? 'd' :
bh_off ? 'b' :
- !IS_ENABLED(CONFIG_TRACE_IRQFLAGS_SUPPORT) ? 'X' :
'.';
switch (entry->flags & (TRACE_FLAG_NEED_RESCHED | TRACE_FLAG_NEED_RESCHED_LAZY |
@@ -1 +1 @@
--rt5
+-rt6