Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commitba1f2b2

Browse files
author
Peter Zijlstra
committed
x86/entry: Fix NMI vs IRQ state tracking
While the nmi_enter() users didtrace_hardirqs_{off_prepare,on_finish}() there was no matchinglockdep_hardirqs_*() calls to complete the picture.Introduce idtentry_{enter,exit}_nmi() to enable proper IRQ statetracking across the NMIs.Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>Reviewed-by: Ingo Molnar <mingo@kernel.org>Link:https://lkml.kernel.org/r/20200623083721.216740948@infradead.org
1 parent859d069 commitba1f2b2

File tree

5 files changed

+70
-29
lines changed

5 files changed

+70
-29
lines changed

‎arch/x86/entry/common.c‎

Lines changed: 38 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -592,7 +592,7 @@ SYSCALL_DEFINE0(ni_syscall)
592592
* The return value must be fed into the state argument of
593593
* idtentry_exit().
594594
*/
595-
idtentry_state_tnoinstridtentry_enter(structpt_regs*regs)
595+
noinstridtentry_state_tidtentry_enter(structpt_regs*regs)
596596
{
597597
idtentry_state_tret= {
598598
.exit_rcu= false,
@@ -687,7 +687,7 @@ static void idtentry_exit_cond_resched(struct pt_regs *regs, bool may_sched)
687687
* Counterpart to idtentry_enter(). The return value of the entry
688688
* function must be fed into the @state argument.
689689
*/
690-
voidnoinstridtentry_exit(structpt_regs*regs,idtentry_state_tstate)
690+
noinstrvoididtentry_exit(structpt_regs*regs,idtentry_state_tstate)
691691
{
692692
lockdep_assert_irqs_disabled();
693693

@@ -731,7 +731,7 @@ void noinstr idtentry_exit(struct pt_regs *regs, idtentry_state_t state)
731731
* Invokes enter_from_user_mode() to establish the proper context for
732732
* NOHZ_FULL. Otherwise scheduling on exit would not be possible.
733733
*/
734-
voidnoinstridtentry_enter_user(structpt_regs*regs)
734+
noinstrvoididtentry_enter_user(structpt_regs*regs)
735735
{
736736
check_user_regs(regs);
737737
enter_from_user_mode();
@@ -749,13 +749,47 @@ void noinstr idtentry_enter_user(struct pt_regs *regs)
749749
*
750750
* Counterpart to idtentry_enter_user().
751751
*/
752-
voidnoinstridtentry_exit_user(structpt_regs*regs)
752+
noinstrvoididtentry_exit_user(structpt_regs*regs)
753753
{
754754
lockdep_assert_irqs_disabled();
755755

756756
prepare_exit_to_usermode(regs);
757757
}
758758

759+
noinstrboolidtentry_enter_nmi(structpt_regs*regs)
760+
{
761+
boolirq_state=lockdep_hardirqs_enabled(current);
762+
763+
__nmi_enter();
764+
lockdep_hardirqs_off(CALLER_ADDR0);
765+
lockdep_hardirq_enter();
766+
rcu_nmi_enter();
767+
768+
instrumentation_begin();
769+
trace_hardirqs_off_finish();
770+
ftrace_nmi_enter();
771+
instrumentation_end();
772+
773+
returnirq_state;
774+
}
775+
776+
noinstrvoididtentry_exit_nmi(structpt_regs*regs,boolrestore)
777+
{
778+
instrumentation_begin();
779+
ftrace_nmi_exit();
780+
if (restore) {
781+
trace_hardirqs_on_prepare();
782+
lockdep_hardirqs_on_prepare(CALLER_ADDR0);
783+
}
784+
instrumentation_end();
785+
786+
rcu_nmi_exit();
787+
lockdep_hardirq_exit();
788+
if (restore)
789+
lockdep_hardirqs_on(CALLER_ADDR0);
790+
__nmi_exit();
791+
}
792+
759793
#ifdefCONFIG_XEN_PV
760794
#ifndefCONFIG_PREEMPTION
761795
/*

‎arch/x86/include/asm/idtentry.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,9 @@ typedef struct idtentry_state {
2020
idtentry_state_tidtentry_enter(structpt_regs*regs);
2121
voididtentry_exit(structpt_regs*regs,idtentry_state_tstate);
2222

23+
boolidtentry_enter_nmi(structpt_regs*regs);
24+
voididtentry_exit_nmi(structpt_regs*regs,boolirq_state);
25+
2326
/**
2427
* DECLARE_IDTENTRY - Declare functions for simple IDT entry points
2528
* No error code pushed by hardware

‎arch/x86/kernel/nmi.c‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -330,7 +330,6 @@ static noinstr void default_do_nmi(struct pt_regs *regs)
330330
__this_cpu_write(last_nmi_rip,regs->ip);
331331

332332
instrumentation_begin();
333-
trace_hardirqs_off_finish();
334333

335334
handled=nmi_handle(NMI_LOCAL,regs);
336335
__this_cpu_add(nmi_stats.normal,handled);
@@ -417,8 +416,6 @@ static noinstr void default_do_nmi(struct pt_regs *regs)
417416
unknown_nmi_error(reason,regs);
418417

419418
out:
420-
if (regs->flags&X86_EFLAGS_IF)
421-
trace_hardirqs_on_prepare();
422419
instrumentation_end();
423420
}
424421

@@ -478,6 +475,8 @@ static DEFINE_PER_CPU(unsigned long, nmi_dr7);
478475

479476
DEFINE_IDTENTRY_RAW(exc_nmi)
480477
{
478+
boolirq_state;
479+
481480
if (IS_ENABLED(CONFIG_SMP)&&arch_cpu_is_offline(smp_processor_id()))
482481
return;
483482

@@ -491,14 +490,14 @@ DEFINE_IDTENTRY_RAW(exc_nmi)
491490

492491
this_cpu_write(nmi_dr7,local_db_save());
493492

494-
nmi_enter();
493+
irq_state=idtentry_enter_nmi(regs);
495494

496495
inc_irq_stat(__nmi_count);
497496

498497
if (!ignore_nmis)
499498
default_do_nmi(regs);
500499

501-
nmi_exit();
500+
idtentry_exit_nmi(regs,irq_state);
502501

503502
local_db_restore(this_cpu_read(nmi_dr7));
504503

‎arch/x86/kernel/traps.c‎

Lines changed: 6 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -403,7 +403,7 @@ DEFINE_IDTENTRY_DF(exc_double_fault)
403403
}
404404
#endif
405405

406-
nmi_enter();
406+
idtentry_enter_nmi(regs);
407407
instrumentation_begin();
408408
notify_die(DIE_TRAP,str,regs,error_code,X86_TRAP_DF,SIGSEGV);
409409

@@ -649,15 +649,12 @@ DEFINE_IDTENTRY_RAW(exc_int3)
649649
instrumentation_end();
650650
idtentry_exit_user(regs);
651651
}else {
652-
nmi_enter();
652+
boolirq_state=idtentry_enter_nmi(regs);
653653
instrumentation_begin();
654-
trace_hardirqs_off_finish();
655654
if (!do_int3(regs))
656655
die("int3",regs,0);
657-
if (regs->flags&X86_EFLAGS_IF)
658-
trace_hardirqs_on_prepare();
659656
instrumentation_end();
660-
nmi_exit();
657+
idtentry_exit_nmi(regs,irq_state);
661658
}
662659
}
663660

@@ -865,9 +862,8 @@ static void handle_debug(struct pt_regs *regs, unsigned long dr6, bool user)
865862
static__always_inlinevoidexc_debug_kernel(structpt_regs*regs,
866863
unsigned longdr6)
867864
{
868-
nmi_enter();
865+
boolirq_state=idtentry_enter_nmi(regs);
869866
instrumentation_begin();
870-
trace_hardirqs_off_finish();
871867

872868
/*
873869
* If something gets miswired and we end up here for a user mode
@@ -884,10 +880,8 @@ static __always_inline void exc_debug_kernel(struct pt_regs *regs,
884880

885881
handle_debug(regs,dr6, false);
886882

887-
if (regs->flags&X86_EFLAGS_IF)
888-
trace_hardirqs_on_prepare();
889883
instrumentation_end();
890-
nmi_exit();
884+
idtentry_exit_nmi(regs,irq_state);
891885
}
892886

893887
static__always_inlinevoidexc_debug_user(structpt_regs*regs,
@@ -903,6 +897,7 @@ static __always_inline void exc_debug_user(struct pt_regs *regs,
903897
instrumentation_begin();
904898

905899
handle_debug(regs,dr6, true);
900+
906901
instrumentation_end();
907902
idtentry_exit_user(regs);
908903
}

‎include/linux/hardirq.h‎

Lines changed: 19 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -111,32 +111,42 @@ extern void rcu_nmi_exit(void);
111111
/*
112112
* nmi_enter() can nest up to 15 times; see NMI_BITS.
113113
*/
114-
#definenmi_enter()\
114+
#define__nmi_enter()\
115115
do {\
116+
lockdep_off();\
116117
arch_nmi_enter();\
117118
printk_nmi_enter();\
118-
lockdep_off();\
119119
BUG_ON(in_nmi() == NMI_MASK);\
120120
__preempt_count_add(NMI_OFFSET + HARDIRQ_OFFSET);\
121-
rcu_nmi_enter();\
121+
} while (0)
122+
123+
#definenmi_enter()\
124+
do {\
125+
__nmi_enter();\
122126
lockdep_hardirq_enter();\
127+
rcu_nmi_enter();\
123128
instrumentation_begin();\
124129
ftrace_nmi_enter();\
125130
instrumentation_end();\
126131
} while (0)
127132

133+
#define__nmi_exit()\
134+
do {\
135+
BUG_ON(!in_nmi());\
136+
__preempt_count_sub(NMI_OFFSET + HARDIRQ_OFFSET);\
137+
printk_nmi_exit();\
138+
arch_nmi_exit();\
139+
lockdep_on();\
140+
} while (0)
141+
128142
#definenmi_exit()\
129143
do {\
130144
instrumentation_begin();\
131145
ftrace_nmi_exit();\
132146
instrumentation_end();\
133-
lockdep_hardirq_exit();\
134147
rcu_nmi_exit();\
135-
BUG_ON(!in_nmi());\
136-
__preempt_count_sub(NMI_OFFSET + HARDIRQ_OFFSET);\
137-
lockdep_on();\
138-
printk_nmi_exit();\
139-
arch_nmi_exit();\
148+
lockdep_hardirq_exit();\
149+
__nmi_exit();\
140150
} while (0)
141151

142152
#endif/* LINUX_HARDIRQ_H */

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp