x86: Put hot per CPU variables into a struct
https://lore.kernel.org/lkml/166601845607.401.14987443239536693764.tip-bot2@tip-bot2/T/
several patches changed the layout of “hot” percpu data.
e.g.
e57ef2ed Thomas Gleixner
x86: Put hot per CPU variables into a struct
64701838 Thomas Gleixner
x86/percpu: Move preempt_count next to current_task
.. importantly:
commit e57ef2ed97c1d078973298658a8096644a1e9e09
Author: Thomas Gleixner <tglx@linutronix.de>
Date: Thu Sep 15 13:11:01 2022 +0200
x86: Put hot per CPU variables into a struct
The layout of per-cpu variables is at the mercy of the compiler. This
can lead to random performance fluctuations from build to build.
Create a structure to hold some of the hottest per-cpu variables,
starting with current_task.
what it does
some hot per cpu stuffs are defined from multiple places
// arch/x86/include/asm/preempt.h
DECLARE_PER_CPU(int, __preempt_count);
// arch/x86/include/asm/current.h
DECLARE_PER_CPU(struct task_struct *, current_task);
// ..
they are packed into struct pcpu_hot
:
// arch/x86/include/asm/current.h
struct pcpu_hot {
union {
struct {
struct task_struct *current_task;
int preempt_count;
int cpu_number;
#ifdef CONFIG_MITIGATION_CALL_DEPTH_TRACKING
u64 call_depth;
#endif
unsigned long top_of_stack;
void *hardirq_stack_ptr;
u16 softirq_pending;
#ifdef CONFIG_X86_64
bool hardirq_stack_inuse;
#else
void *softirq_stack_ptr;
#endif
};
u8 pad[64];
};
};
DECLARE_PER_CPU_ALIGNED(struct pcpu_hot, pcpu_hot);
/* const-qualified alias to pcpu_hot, aliased by linker. */
DECLARE_PER_CPU_ALIGNED(const struct pcpu_hot __percpu_seg_override,
const_pcpu_hot);
and
// arch/x86/kernel/cpu/common.c
DEFINE_PER_CPU_ALIGNED(struct pcpu_hot, pcpu_hot) = {
.current_task = &init_task,
.preempt_count = INIT_PREEMPT_COUNT,
.top_of_stack = TOP_OF_INIT_STACK,
};
EXPORT_PER_CPU_SYMBOL(pcpu_hot);
EXPORT_PER_CPU_SYMBOL(const_pcpu_hot);
access, through the pcpu struct
- per_cpu(current_task, cpu) = idle;
+ per_cpu(pcpu_hot.current_task, cpu) = idle;