[SRU][Xenial][PATCH v2 2/4] UBUNTU: SAUCE: x86/speculation: Cleanup IBRS runtime control handling
Kleber Souza
kleber.souza at canonical.com
Thu Jan 10 09:48:12 UTC 2019
On 12/13/18 2:21 PM, Juerg Haefliger wrote:
> In Ubuntu, we have runtime control for enabling/disabling IBRS via the
> commandline ("noibrs") and through the proc interface
> /proc/sys/kernel/ibrs_enabled. This commit simplifies the current
> (probably broken) implementation by merging it with all the IBRS-related
> upstream changes from previous commits.
>
> What we have now is the upstream implementation for detecting the presence
> of IBRS support. This commit adds a global state variable 'ibrs_enabled'
> which is set to 1 if the CPU supports IBRS but can be overridden via the
> commandline "noibrs" switch or by writting 0, 1 or 2 to
> /proc/sys/kernel/ibrs_enabled at runtime.
>
> Note that the runtime controls are disabled if the CPU runs in Enhanced
> IBRS mode.
>
> CVE-2017-5715
>
> Signed-off-by: Juerg Haefliger <juergh at canonical.com>
> ---
> arch/x86/include/asm/mwait.h | 6 +-
> arch/x86/include/asm/nospec-branch.h | 20 +++++
> arch/x86/include/asm/spec_ctrl.h | 10 ++-
> arch/x86/kernel/cpu/bugs.c | 45 ++++-------
> arch/x86/kernel/cpu/microcode/core.c | 11 ---
> arch/x86/kernel/process.c | 10 +--
> arch/x86/kernel/smpboot.c | 6 +-
> arch/x86/lib/delay.c | 8 +-
> include/linux/smp.h | 46 -----------
> kernel/smp.c | 28 -------
> kernel/sysctl.c | 115 ++++++++++++++++-----------
> 11 files changed, 123 insertions(+), 182 deletions(-)
>
> diff --git a/arch/x86/include/asm/mwait.h b/arch/x86/include/asm/mwait.h
> index 9821763b02cf..9bd760758640 100644
> --- a/arch/x86/include/asm/mwait.h
> +++ b/arch/x86/include/asm/mwait.h
> @@ -106,15 +106,13 @@ static inline void mwait_idle_with_hints(unsigned long eax, unsigned long ecx)
> mb();
> }
>
> - if (ibrs_inuse)
> - native_wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
> + ubuntu_restrict_branch_speculation_end();
>
> __monitor((void *)¤t_thread_info()->flags, 0, 0);
> if (!need_resched())
> __mwait(eax, ecx);
>
> - if (ibrs_inuse)
> - native_wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base | SPEC_CTRL_IBRS);
> + ubuntu_restrict_branch_speculation_start();
> }
> current_clr_polling();
> }
> diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h
> index dcc7b0348fbc..a6120d43caa7 100644
> --- a/arch/x86/include/asm/nospec-branch.h
> +++ b/arch/x86/include/asm/nospec-branch.h
> @@ -188,6 +188,10 @@
> extern unsigned int ibpb_enabled;
> int set_ibpb_enabled(unsigned int);
>
> +/* The IBRS runtime control knob */
> +extern unsigned int ibrs_enabled;
> +int set_ibrs_enabled(unsigned int);
> +
> /* The Spectre V2 mitigation variants */
> enum spectre_v2_mitigation {
> SPECTRE_V2_NONE,
> @@ -275,6 +279,22 @@ do { \
> preempt_enable(); \
> } while (0)
>
> +#define ubuntu_restrict_branch_speculation_start() \
> +do { \
> + u64 val = x86_spec_ctrl_base | SPEC_CTRL_IBRS; \
> + \
> + if (ibrs_enabled) \
> + native_wrmsrl(MSR_IA32_SPEC_CTRL, val); \
> +} while (0)
> +
> +#define ubuntu_restrict_branch_speculation_end() \
> +do { \
> + u64 val = x86_spec_ctrl_base; \
> + \
> + if (ibrs_enabled) \
> + native_wrmsrl(MSR_IA32_SPEC_CTRL, val); \
> + } while (0)
> +
> #endif /* __ASSEMBLY__ */
>
> /*
> diff --git a/arch/x86/include/asm/spec_ctrl.h b/arch/x86/include/asm/spec_ctrl.h
> index 49c3b0a83e9f..a5d93d23390e 100644
> --- a/arch/x86/include/asm/spec_ctrl.h
> +++ b/arch/x86/include/asm/spec_ctrl.h
> @@ -8,7 +8,7 @@
>
> #ifdef __ASSEMBLY__
>
> -.extern use_ibrs
> +.extern ibrs_enabled
>
> #define __ASM_ENABLE_IBRS \
> pushq %rax; \
> @@ -21,11 +21,13 @@
> popq %rdx; \
> popq %rcx; \
> popq %rax
> +
> #define __ASM_ENABLE_IBRS_CLOBBER \
> movl $MSR_IA32_SPEC_CTRL, %ecx; \
> movl $0, %edx; \
> movl $SPEC_CTRL_IBRS, %eax; \
> wrmsr;
> +
> #define __ASM_DISABLE_IBRS \
> pushq %rax; \
> pushq %rcx; \
> @@ -39,7 +41,7 @@
> popq %rax
>
> .macro ENABLE_IBRS
> - testl $1, use_ibrs
> + testl $1, ibrs_enabled
> jz 10f
> __ASM_ENABLE_IBRS
> jmp 20f
> @@ -49,7 +51,7 @@
> .endm
>
> .macro ENABLE_IBRS_CLOBBER
> - testl $1, use_ibrs
> + testl $1, ibrs_enabled
> jz 11f
> __ASM_ENABLE_IBRS_CLOBBER
> jmp 21f
> @@ -59,7 +61,7 @@
> .endm
>
> .macro DISABLE_IBRS
> - testl $1, use_ibrs
> + testl $1, ibrs_enabled
> jz 9f
> __ASM_DISABLE_IBRS
> 9:
> diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c
> index eeb89d781a9c..7a9940015af5 100644
> --- a/arch/x86/kernel/cpu/bugs.c
> +++ b/arch/x86/kernel/cpu/bugs.c
> @@ -40,6 +40,16 @@ static int __init noibpb_handler(char *str)
>
> early_param("noibpb", noibpb_handler);
>
> +unsigned int noibrs = 0;
> +
> +static int __init noibrs_handler(char *str)
> +{
> + noibrs = 1;
> + return 0;
> +}
> +
> +early_param("noibrs", noibrs_handler);
> +
> static void __init spectre_v2_select_mitigation(void);
> static void __init ssb_select_mitigation(void);
> static void __init l1tf_select_mitigation(void);
> @@ -410,18 +420,6 @@ specv2_set_mode:
> set_ibpb_enabled(1); /* Enable IBPB */
> }
>
> - /* Initialize Indirect Branch Restricted Speculation if supported */
> - if (boot_cpu_has(X86_FEATURE_IBRS)) {
> - pr_info("Spectre v2 mitigation: Enabling Indirect Branch Restricted Speculation\n");
> -
> - set_ibrs_supported();
> - if (ibrs_inuse)
> - sysctl_ibrs_enabled = 1;
> - }
> -
> - pr_info("Spectre v2 mitigation: Speculation control IBRS %s",
> - ibrs_supported ? "supported" : "not-supported");
> -
> /*
> * If spectre v2 protection has been enabled, unconditionally fill
> * RSB during a context switch; this protects against two independent
> @@ -433,21 +431,6 @@ specv2_set_mode:
> setup_force_cpu_cap(X86_FEATURE_RSB_CTXSW);
> pr_info("Spectre v2 / SpectreRSB mitigation: Filling RSB on context switch\n");
>
> - /*
> - * If we have a full retpoline mode and then disable IBPB in kernel mode
> - * we do not require both.
> - */
> - if (mode == SPECTRE_V2_RETPOLINE_AMD ||
> - mode == SPECTRE_V2_RETPOLINE_GENERIC)
> - {
> - if (ibrs_supported) {
> - pr_info("Retpoline compiled kernel. Defaulting IBRS to disabled");
> - set_ibrs_disabled();
> - if (!ibrs_inuse)
> - sysctl_ibrs_enabled = 0;
> - }
> - }
> -
> /*
> * Retpoline means the kernel is safe because it has no indirect
> * branches. Enhanced IBRS protects firmware too, so, enable restricted
> @@ -462,6 +445,11 @@ specv2_set_mode:
> if (boot_cpu_has(X86_FEATURE_IBRS) && mode != SPECTRE_V2_IBRS_ENHANCED) {
> setup_force_cpu_cap(X86_FEATURE_USE_IBRS_FW);
> pr_info("Enabling Restricted Speculation for firmware calls\n");
> +
> + if (!noibrs &&
> + mode != SPECTRE_V2_RETPOLINE_GENERIC &&
> + mode != SPECTRE_V2_RETPOLINE_AMD)
> + set_ibrs_enabled(1); /* Enable IBRS */
> }
> }
>
> @@ -871,8 +859,9 @@ static ssize_t cpu_show_common(struct device *dev, struct device_attribute *attr
> break;
>
> case X86_BUG_SPECTRE_V2:
> - return sprintf(buf, "%s%s%s%s\n", spectre_v2_strings[spectre_v2_enabled],
> + return sprintf(buf, "%s%s%s%s%s\n", spectre_v2_strings[spectre_v2_enabled],
> ibpb_enabled ? ", IBPB" : "",
> + ibrs_enabled == 2 ? ", IBRS (user space)" : ibrs_enabled ? ", IBRS" : "",
> boot_cpu_has(X86_FEATURE_USE_IBRS_FW) ? ", IBRS_FW" : "",
> spectre_v2_module_string());
>
> diff --git a/arch/x86/kernel/cpu/microcode/core.c b/arch/x86/kernel/cpu/microcode/core.c
> index 84bd97f8eeab..4fe475c2f745 100644
> --- a/arch/x86/kernel/cpu/microcode/core.c
> +++ b/arch/x86/kernel/cpu/microcode/core.c
> @@ -439,17 +439,6 @@ static ssize_t reload_store(struct device *dev,
> if (!ret)
> perf_check_microcode();
>
> - /* Initialize Indirect Branch Restricted Speculation if supported */
> - if (boot_cpu_has(X86_FEATURE_IBRS)) {
> - pr_info("Enabling Indirect Branch Restricted Speculation\n");
> -
> - mutex_lock(&spec_ctrl_mutex);
> - set_ibrs_supported();
> - if (ibrs_inuse)
> - sysctl_ibrs_enabled = 1;
> - mutex_unlock(&spec_ctrl_mutex);
> - }
> -
> mutex_unlock(µcode_mutex);
> put_online_cpus();
>
> diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c
> index a5b3404266eb..5587098a5823 100644
> --- a/arch/x86/kernel/process.c
> +++ b/arch/x86/kernel/process.c
> @@ -566,17 +566,15 @@ static void mwait_idle(void)
> smp_mb(); /* quirk */
> }
>
> - if (ibrs_inuse)
> - native_wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
> + ubuntu_restrict_branch_speculation_end();
>
> __monitor((void *)¤t_thread_info()->flags, 0, 0);
> +
> if (!need_resched()) {
> __sti_mwait(0, 0);
> - if (ibrs_inuse)
> - native_wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base | SPEC_CTRL_IBRS);
> + ubuntu_restrict_branch_speculation_start();
> } else {
> - if (ibrs_inuse)
> - native_wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base | SPEC_CTRL_IBRS);
> + ubuntu_restrict_branch_speculation_start();
> local_irq_enable();
> }
> trace_cpu_idle_rcuidle(PWR_EVENT_EXIT, smp_processor_id());
> diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c
> index 21c3bcbd69a2..6c9bb4db2ed7 100644
> --- a/arch/x86/kernel/smpboot.c
> +++ b/arch/x86/kernel/smpboot.c
> @@ -1697,15 +1697,13 @@ void native_play_dead(void)
> play_dead_common();
> tboot_shutdown(TB_SHUTDOWN_WFS);
>
> - if (ibrs_inuse)
> - native_wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
> + ubuntu_restrict_branch_speculation_end();
>
> mwait_play_dead(); /* Only returns on failure */
> if (cpuidle_play_dead())
> hlt_play_dead();
>
> - if (ibrs_inuse)
> - native_wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base | SPEC_CTRL_IBRS);
> + ubuntu_restrict_branch_speculation_start();
> }
>
> #else /* ... !CONFIG_HOTPLUG_CPU */
> diff --git a/arch/x86/lib/delay.c b/arch/x86/lib/delay.c
> index 5b66d4417f8d..20e9a5ac91dc 100644
> --- a/arch/x86/lib/delay.c
> +++ b/arch/x86/lib/delay.c
> @@ -107,8 +107,8 @@ static void delay_mwaitx(unsigned long __loops)
> for (;;) {
> delay = min_t(u64, MWAITX_MAX_LOOPS, loops);
>
> - if (ibrs_inuse && (delay > IBRS_DISABLE_THRESHOLD))
> - native_wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
> + if (delay > IBRS_DISABLE_THRESHOLD)
> + ubuntu_restrict_branch_speculation_end();
>
> /*
> * Use cpu_tss as a cacheline-aligned, seldomly
> @@ -123,8 +123,8 @@ static void delay_mwaitx(unsigned long __loops)
> */
> __mwaitx(MWAITX_DISABLE_CSTATES, delay, MWAITX_ECX_TIMER_ENABLE);
>
> - if (ibrs_inuse && (delay > IBRS_DISABLE_THRESHOLD))
> - native_wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base | SPEC_CTRL_IBRS);
> + if (delay > IBRS_DISABLE_THRESHOLD)
> + ubuntu_restrict_branch_speculation_start();
>
> end = rdtsc_ordered();
>
> diff --git a/include/linux/smp.h b/include/linux/smp.h
> index 98ac8ff2afab..c4414074bd88 100644
> --- a/include/linux/smp.h
> +++ b/include/linux/smp.h
> @@ -50,52 +50,6 @@ void on_each_cpu_cond(bool (*cond_func)(int cpu, void *info),
>
> int smp_call_function_single_async(int cpu, struct call_single_data *csd);
>
> -#ifdef CONFIG_X86
> -
> -#define IBxx_INUSE (1 << 0)
> -#define IBxx_SUPPORTED (1 << 1)
> -#define IBxx_DISABLED (1 << 2)
> -
> -/* indicate usage of IBRS to control execution speculation */
> -extern int use_ibrs;
> -extern u32 sysctl_ibrs_enabled;
> -extern struct mutex spec_ctrl_mutex;
> -
> -static inline int __check_ibrs_inuse(void)
> -{
> - if (use_ibrs & IBxx_INUSE)
> - return 1;
> - else
> - /* rmb to prevent wrong speculation for security */
> - rmb();
> - return 0;
> -}
> -
> -#define ibrs_inuse (__check_ibrs_inuse())
> -#define ibrs_supported (use_ibrs & IBxx_SUPPORTED)
> -#define ibrs_disabled (use_ibrs & IBxx_DISABLED)
> -
> -static inline void set_ibrs_supported(void)
> -{
> - use_ibrs |= IBxx_SUPPORTED;
> - if (!ibrs_disabled)
> - use_ibrs |= IBxx_INUSE;
> -}
> -static inline void set_ibrs_disabled(void)
> -{
> - use_ibrs |= IBxx_DISABLED;
> - if (ibrs_inuse)
> - use_ibrs &= ~IBxx_INUSE;
> -}
> -static inline void clear_ibrs_disabled(void)
> -{
> - use_ibrs &= ~IBxx_DISABLED;
> - if (ibrs_supported)
> - use_ibrs |= IBxx_INUSE;
> -}
> -
> -#endif /* CONFIG_X86 */
> -
> #ifdef CONFIG_SMP
>
> #include <linux/preempt.h>
> diff --git a/kernel/smp.c b/kernel/smp.c
> index 139681ddf916..48eb4fafc356 100644
> --- a/kernel/smp.c
> +++ b/kernel/smp.c
> @@ -499,22 +499,6 @@ EXPORT_SYMBOL(smp_call_function);
> unsigned int setup_max_cpus = NR_CPUS;
> EXPORT_SYMBOL(setup_max_cpus);
>
> -#ifdef CONFIG_X86
> -/*
> - * use IBRS
> - * bit 0 = indicate if ibrs is currently in use
> - * bit 1 = indicate if system supports ibrs
> - * bit 2 = indicate if admin disables ibrs
> -*/
> -
> -int use_ibrs;
> -EXPORT_SYMBOL(use_ibrs);
> -#endif
> -
> -/* mutex to serialize IBRS & IBPB control changes */
> -DEFINE_MUTEX(spec_ctrl_mutex);
> -EXPORT_SYMBOL(spec_ctrl_mutex);
> -
> /*
> * Setup routine for controlling SMP activation
> *
> @@ -538,18 +522,6 @@ static int __init nosmp(char *str)
>
> early_param("nosmp", nosmp);
>
> -#ifdef CONFIG_X86
> -static int __init noibrs(char *str)
> -{
> - set_ibrs_disabled();
> -
> - return 0;
> -}
> -
> -early_param("noibrs", noibrs);
> -#endif
> -
> -
> /* this is hard limit */
> static int __init nrcpus(char *str)
> {
> diff --git a/kernel/sysctl.c b/kernel/sysctl.c
> index d00f95726ac4..02df9c031a45 100644
> --- a/kernel/sysctl.c
> +++ b/kernel/sysctl.c
> @@ -202,8 +202,8 @@ static int proc_dostring_coredump(struct ctl_table *table, int write,
> #endif
>
> #ifdef CONFIG_X86
> -int proc_dointvec_ibrs_ctrl(struct ctl_table *table, int write,
> - void __user *buffer, size_t *lenp, loff_t *ppos);
> +/* Mutex to serialize IBPB and IBRS runtime control changes */
> +DEFINE_MUTEX(spec_ctrl_mutex);
>
> unsigned int ibpb_enabled = 0;
> EXPORT_SYMBOL(ibpb_enabled); /* Required in some modules */
> @@ -252,6 +252,70 @@ static int ibpb_enabled_handler(struct ctl_table *table, int write,
>
> return set_ibpb_enabled(__ibpb_enabled);
> }
> +
> +unsigned int ibrs_enabled = 0;
> +EXPORT_SYMBOL(ibrs_enabled); /* Required in some modules */
> +
> +static unsigned int __ibrs_enabled = 0; /* procfs shadow variable */
> +
> +int set_ibrs_enabled(unsigned int val)
> +{
> + int error = 0;
> + unsigned int cpu;
> + unsigned int prev = ibrs_enabled;
> +
> + mutex_lock(&spec_ctrl_mutex);
> +
> + /* Only enable/disable IBRS if the CPU supports it */
> + if (boot_cpu_has(X86_FEATURE_USE_IBRS_FW)) {
> + ibrs_enabled = val;
> + if (ibrs_enabled != prev)
> + pr_info("Spectre V2 : Spectre v2 mitigation: %s "
> + "Indirect Branch Restricted Speculation%s\n",
> + ibrs_enabled ? "Enabling" : "Disabling",
> + ibrs_enabled == 2 ? " (user space)" : "");
> +
> + if (ibrs_enabled == 0) {
> + /* Always disable IBRS */
> + u64 val = x86_spec_ctrl_base;
> +
> + for_each_online_cpu(cpu)
> + wrmsrl_on_cpu(cpu, MSR_IA32_SPEC_CTRL, val);
We don't need to handle here the case ibrs_enabled == 1?
> + } else if (ibrs_enabled == 2) {
> + /* Always enable IBRS, even in user space */
> + u64 val = x86_spec_ctrl_base | SPEC_CTRL_IBRS;
> +
> + for_each_online_cpu(cpu)
> + wrmsrl_on_cpu(cpu, MSR_IA32_SPEC_CTRL, val);
> + }
> + } else {
> + ibrs_enabled = 0;
> + if (val) {
> + /* IBRS is not supported but we try to turn it on */
> + error = -EINVAL;
> + }
> + }
> +
> + /* Update the shadow variable */
> + __ibrs_enabled = ibrs_enabled;
> +
> + mutex_unlock(&spec_ctrl_mutex);
> +
> + return error;
> +}
> +
> +static int ibrs_enabled_handler(struct ctl_table *table, int write,
> + void __user *buffer, size_t *lenp,
> + loff_t *ppos)
> +{
> + int error;
> +
> + error = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
> + if (error)
> + return error;
> +
> + return set_ibrs_enabled(__ibrs_enabled);
> +}
> #endif
>
> #ifdef CONFIG_MAGIC_SYSRQ
> @@ -290,9 +354,6 @@ extern struct ctl_table epoll_table[];
> int sysctl_legacy_va_layout;
> #endif
>
> -u32 sysctl_ibrs_enabled = 0;
> -EXPORT_SYMBOL(sysctl_ibrs_enabled);
> -
> /* The default sysctl tables: */
>
> static struct ctl_table sysctl_base_table[] = {
> @@ -1281,10 +1342,10 @@ static struct ctl_table kern_table[] = {
> #ifdef CONFIG_X86
> {
> .procname = "ibrs_enabled",
> - .data = &sysctl_ibrs_enabled,
> + .data = &__ibrs_enabled,
> .maxlen = sizeof(unsigned int),
> .mode = 0644,
> - .proc_handler = proc_dointvec_ibrs_ctrl,
> + .proc_handler = ibrs_enabled_handler,
> .extra1 = &zero,
> .extra2 = &two,
> },
> @@ -2455,46 +2516,6 @@ int proc_dointvec_minmax(struct ctl_table *table, int write,
> do_proc_dointvec_minmax_conv, ¶m);
> }
>
> -#ifdef CONFIG_X86
> -int proc_dointvec_ibrs_ctrl(struct ctl_table *table, int write,
> - void __user *buffer, size_t *lenp, loff_t *ppos)
> -{
> - int ret;
> - unsigned int cpu;
> -
> - ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
> - pr_debug("sysctl_ibrs_enabled = %u\n", sysctl_ibrs_enabled);
> - pr_debug("before:use_ibrs = %d\n", use_ibrs);
> - mutex_lock(&spec_ctrl_mutex);
> - if (sysctl_ibrs_enabled == 0) {
> - /* always set IBRS off */
> - set_ibrs_disabled();
> - if (ibrs_supported) {
> - for_each_online_cpu(cpu)
> - wrmsrl_on_cpu(cpu, MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
> - }
> - } else if (sysctl_ibrs_enabled == 2) {
> - /* always set IBRS on, even in user space */
> - clear_ibrs_disabled();
> - if (ibrs_supported) {
> - for_each_online_cpu(cpu)
> - wrmsrl_on_cpu(cpu, MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base | SPEC_CTRL_IBRS);
> - } else {
> - sysctl_ibrs_enabled = 0;
> - }
> - } else if (sysctl_ibrs_enabled == 1) {
> - /* use IBRS in kernel */
> - clear_ibrs_disabled();
> - if (!ibrs_inuse)
> - /* platform don't support ibrs */
> - sysctl_ibrs_enabled = 0;
> - }
> - mutex_unlock(&spec_ctrl_mutex);
> - pr_debug("after:use_ibrs = %d\n", use_ibrs);
> - return ret;
> -}
> -#endif
> -
> static void validate_coredump_safety(void)
> {
> #ifdef CONFIG_COREDUMP
More information about the kernel-team
mailing list