Date: Sat, 27 Jan 2018 11:19:41 +0000 (UTC) From: Michal Meloun <mmel@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r328467 - in head/sys/arm: arm include Message-ID: <201801271119.w0RBJfD3027819@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: mmel Date: Sat Jan 27 11:19:41 2018 New Revision: 328467 URL: https://svnweb.freebsd.org/changeset/base/328467 Log: Implement mitigation for Spectre version 2 attacks on ARMv7. Similarly as we already do for arm64, for mitigation is necessary to flush branch predictor when we: - do task switch - receive prefetch abort on non-userspace address The user can disable this mitigation by setting 'machdep.disable_bp_hardening' sysctl variable, or it can check actual system status by reading 'machdep.spectre_v2_safe' The situation is complicated by fact that: - for Cortex-A8, the BPIALL instruction is effectively NOP until the IBE bit in ACTLR is set. - for Cortex-A15, the BPIALL is always NOP. The branch predictor can be only flushed by doing ICIALLU with special bit (Enable invalidates of BTB) set in ACTLR. Since access to the ACTLR register is locked to secure monitor/firmware on most boards, they will also need update of firmware / U-boot. In worst case, when secure monitor is on-chip ROM (e.g. PandaBoard), the board is unfixable. MFC after: 2 weeks Reviewed by: imp, emaste Differential Revision: https://reviews.freebsd.org/D13931 Modified: head/sys/arm/arm/cpuinfo.c head/sys/arm/arm/genassym.c head/sys/arm/arm/machdep.c head/sys/arm/arm/mp_machdep.c head/sys/arm/arm/swtch-v6.S head/sys/arm/arm/trap-v6.c head/sys/arm/include/cpuinfo.h head/sys/arm/include/pcpu.h Modified: head/sys/arm/arm/cpuinfo.c ============================================================================== --- head/sys/arm/arm/cpuinfo.c Sat Jan 27 09:49:47 2018 (r328466) +++ head/sys/arm/arm/cpuinfo.c Sat Jan 27 11:19:41 2018 (r328467) @@ -31,6 +31,8 @@ __FBSDID("$FreeBSD$"); #include <sys/param.h> #include <sys/systm.h> #include <sys/kernel.h> +#include <sys/pcpu.h> +#include <sys/smp.h> #include <sys/sysctl.h> #include <machine/cpu.h> @@ -40,6 +42,9 @@ __FBSDID("$FreeBSD$"); #if __ARM_ARCH >= 6 void reinit_mmu(uint32_t ttb, uint32_t aux_clr, uint32_t aux_set); + +int disable_bp_hardening; +int spectre_v2_safe = 1; #endif struct cpuinfo cpuinfo = @@ -255,6 +260,7 @@ cpuinfo_get_actlr_modifier(uint32_t *actlr_mask, uint3 if (cpuinfo.implementer == CPU_IMPLEMENTER_ARM) { switch (cpuinfo.part_number) { + case CPU_ARCH_CORTEX_A75: case CPU_ARCH_CORTEX_A73: case CPU_ARCH_CORTEX_A72: case CPU_ARCH_CORTEX_A57: @@ -337,5 +343,198 @@ cpuinfo_reinit_mmu(uint32_t ttb) actlr_set |= cpu_quirks_actlr_set; reinit_mmu(ttb, actlr_mask, actlr_set); } + +static bool +modify_actlr(uint32_t clear, uint32_t set) +{ + uint32_t reg, newreg; + + reg = cp15_actlr_get(); + newreg = reg; + newreg &= ~clear; + newreg |= set; + if (reg == newreg) + return (true); + cp15_actlr_set(newreg); + + reg = cp15_actlr_get(); + if (reg == newreg) + return (true); + return (false); +} + +/* Apply/restore BP hardening on current core. */ +static int +apply_bp_hardening(bool enable, int kind, bool actrl, uint32_t set_mask) +{ + if (enable) { + if (actrl && !modify_actlr(0, set_mask)) + return (-1); + PCPU_SET(bp_harden_kind, kind); + } else { + PCPU_SET(bp_harden_kind, PCPU_BP_HARDEN_KIND_NONE); + if (actrl) + modify_actlr(~0, PCPU_GET(original_actlr)); + spectre_v2_safe = 0; + } + return (0); +} + +static void +handle_bp_hardening(bool enable) +{ + int kind; + char *kind_str; + + kind = PCPU_BP_HARDEN_KIND_NONE; + /* + * Note: Access to ACTRL is locked to secure world on most boards. + * This means that full BP hardening depends on updated u-boot/firmware + * or is impossible at all (if secure monitor is in on-chip ROM). + */ + if (cpuinfo.implementer == CPU_IMPLEMENTER_ARM) { + switch (cpuinfo.part_number) { + case CPU_ARCH_CORTEX_A8: + /* + * For Cortex-A8, IBE bit must be set otherwise + * BPIALL is effectively NOP. + * Unfortunately, Cortex-A is also affected by + * ARM erratum 687067 which causes non-working + * BPIALL if IBE bit is set and 'Instruction L1 System + * Array Debug Register 0' is not 0. + * This register is not reset on power-up and is + * accessible only from secure world, so we cannot do + * nothing (nor detect) to fix this issue. + * I afraid that on chip ROM based secure monitor on + * AM335x (BeagleBone) doesn't reset this debug + * register. + */ + kind = PCPU_BP_HARDEN_KIND_BPIALL; + if (apply_bp_hardening(enable, kind, true, 1 << 6) != 0) + goto actlr_err; + break; + break; + + case CPU_ARCH_CORTEX_A9: + case CPU_ARCH_CORTEX_A12: + case CPU_ARCH_CORTEX_A17: + case CPU_ARCH_CORTEX_A57: + case CPU_ARCH_CORTEX_A72: + case CPU_ARCH_CORTEX_A73: + case CPU_ARCH_CORTEX_A75: + kind = PCPU_BP_HARDEN_KIND_BPIALL; + if (apply_bp_hardening(enable, kind, false, 0) != 0) + goto actlr_err; + break; + + case CPU_ARCH_CORTEX_A15: + /* + * For Cortex-A15, set 'Enable invalidates of BTB' bit. + * Despite this, the BPIALL is still effectively NOP, + * but with this bit set, the ICIALLU also flushes + * branch predictor as side effect. + */ + kind = PCPU_BP_HARDEN_KIND_ICIALLU; + if (apply_bp_hardening(enable, kind, true, 1 << 0) != 0) + goto actlr_err; + break; + + default: + break; + } + } else if (cpuinfo.implementer == CPU_IMPLEMENTER_QCOM) { + printf("!!!WARNING!!! CPU(%d) is vulnerable to speculative " + "branch attacks. !!!\n" + "Qualcomm Krait cores are known (or believed) to be " + "vulnerable to \n" + "speculative branch attacks, no mitigation exists yet.\n", + PCPU_GET(cpuid)); + goto unkonown_mitigation; + } else { + goto unkonown_mitigation; + } + + if (bootverbose) { + switch (kind) { + case PCPU_BP_HARDEN_KIND_NONE: + kind_str = "not necessary"; + break; + case PCPU_BP_HARDEN_KIND_BPIALL: + kind_str = "BPIALL"; + break; + case PCPU_BP_HARDEN_KIND_ICIALLU: + kind_str = "ICIALLU"; + break; + default: + panic("Unknown BP hardering kind (%d).", kind); + } + printf("CPU(%d) applied BP hardening: %s\n", PCPU_GET(cpuid), + kind_str); + } + + return; + +unkonown_mitigation: + PCPU_SET(bp_harden_kind, PCPU_BP_HARDEN_KIND_NONE); + spectre_v2_safe = 0; + return; + +actlr_err: + PCPU_SET(bp_harden_kind, PCPU_BP_HARDEN_KIND_NONE); + spectre_v2_safe = 0; + printf("!!!WARNING!!! CPU(%d) is vulnerable to speculative branch " + "attacks. !!!\n" + "We cannot enable required bit(s) in ACTRL register\n" + "because it's locked by secure monitor and/or firmware.\n", + PCPU_GET(cpuid)); +} + +void +cpuinfo_init_bp_hardening(void) +{ + + /* + * Store original unmodified ACTRL, so we can restore it when + * BP hardening is disabled by sysctl. + */ + PCPU_SET(original_actlr, cp15_actlr_get()); + handle_bp_hardening(true); +} + +static void +bp_hardening_action(void *arg) +{ + + handle_bp_hardening(disable_bp_hardening == 0); +} + +static int +sysctl_disable_bp_hardening(SYSCTL_HANDLER_ARGS) +{ + int rv; + + rv = sysctl_handle_int(oidp, oidp->oid_arg1, oidp->oid_arg2, req); + + if (!rv && req->newptr) { + spectre_v2_safe = 1; + dmb(); +#ifdef SMP + smp_rendezvous_cpus(all_cpus, smp_no_rendezvous_barrier, + bp_hardening_action, NULL, NULL); +#else + bp_hardening_action(NULL); +#endif + } + + return (rv); +} + +SYSCTL_PROC(_machdep, OID_AUTO, disable_bp_hardening, + CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, + &disable_bp_hardening, 0, sysctl_disable_bp_hardening, "I", + "Disable BP hardening mitigation."); + +SYSCTL_INT(_machdep, OID_AUTO, spectre_v2_safe, CTLFLAG_RD, + &spectre_v2_safe, 0, "System is safe to Spectre Version 2 attacks"); #endif /* __ARM_ARCH >= 6 */ Modified: head/sys/arm/arm/genassym.c ============================================================================== --- head/sys/arm/arm/genassym.c Sat Jan 27 09:49:47 2018 (r328466) +++ head/sys/arm/arm/genassym.c Sat Jan 27 11:19:41 2018 (r328467) @@ -33,6 +33,7 @@ __FBSDID("$FreeBSD$"); #include <sys/cpuset.h> #include <sys/systm.h> #include <sys/assym.h> +#include <sys/pcpu.h> #include <sys/proc.h> #include <sys/mbuf.h> #include <sys/vmmeter.h> @@ -136,6 +137,10 @@ ASSYM(PCB_VFPSTATE, offsetof(struct pcb, pcb_vfpstate) #if __ARM_ARCH >= 6 ASSYM(PC_CURPMAP, offsetof(struct pcpu, pc_curpmap)); +ASSYM(PC_BP_HARDEN_KIND, offsetof(struct pcpu, pc_bp_harden_kind)); +ASSYM(PCPU_BP_HARDEN_KIND_NONE, PCPU_BP_HARDEN_KIND_NONE); +ASSYM(PCPU_BP_HARDEN_KIND_BPIALL, PCPU_BP_HARDEN_KIND_BPIALL); +ASSYM(PCPU_BP_HARDEN_KIND_ICIALLU, PCPU_BP_HARDEN_KIND_ICIALLU); #endif ASSYM(PAGE_SIZE, PAGE_SIZE); Modified: head/sys/arm/arm/machdep.c ============================================================================== --- head/sys/arm/arm/machdep.c Sat Jan 27 09:49:47 2018 (r328466) +++ head/sys/arm/arm/machdep.c Sat Jan 27 11:19:41 2018 (r328467) @@ -1264,6 +1264,8 @@ initarm(struct arm_boot_params *abp) msgbufinit(msgbufp, msgbufsize); dbg_monitor_init(); kdb_init(); + /* Apply possible BP hardening. */ + cpuinfo_init_bp_hardening(); return ((void *)STACKALIGN(thread0.td_pcb)); } Modified: head/sys/arm/arm/mp_machdep.c ============================================================================== --- head/sys/arm/arm/mp_machdep.c Sat Jan 27 09:49:47 2018 (r328466) +++ head/sys/arm/arm/mp_machdep.c Sat Jan 27 11:19:41 2018 (r328467) @@ -201,6 +201,9 @@ init_secondary(int cpu) /* Configure the interrupt controller */ intr_pic_init_secondary(); + /* Apply possible BP hardening */ + cpuinfo_init_bp_hardening(); + mtx_lock_spin(&ap_boot_mtx); atomic_add_rel_32(&smp_cpus, 1); Modified: head/sys/arm/arm/swtch-v6.S ============================================================================== --- head/sys/arm/arm/swtch-v6.S Sat Jan 27 09:49:47 2018 (r328466) +++ head/sys/arm/arm/swtch-v6.S Sat Jan 27 11:19:41 2018 (r328467) @@ -145,7 +145,16 @@ ENTRY(cpu_context_switch) * predictors and Requirements for branch predictor maintenance * operations sections. */ - mcr CP15_BPIALL /* flush entire Branch Target Cache */ + /* + * Additionally, to mitigate mistrained branch predictor attack + * we must invalidate it on affected CPUs. Unfortunately, BPIALL + * is effectively NOP on Cortex-A15 so it needs special treatment. + */ + ldr r0, [r8, #PC_BP_HARDEN_KIND] + cmp r0, #PCPU_BP_HARDEN_KIND_ICIALLU + mcrne CP15_BPIALL /* Flush entire Branch Target Cache */ + mcreq CP15_ICIALLU /* This is the only way how to flush */ + /* Branch Target Cache on Cortex-A15. */ DSB mov pc, lr END(cpu_context_switch) Modified: head/sys/arm/arm/trap-v6.c ============================================================================== --- head/sys/arm/arm/trap-v6.c Sat Jan 27 09:49:47 2018 (r328466) +++ head/sys/arm/arm/trap-v6.c Sat Jan 27 11:19:41 2018 (r328467) @@ -287,6 +287,7 @@ abort_handler(struct trapframe *tf, int prefetch) struct vmspace *vm; vm_prot_t ftype; bool usermode; + int bp_harden; #ifdef INVARIANTS void *onfault; #endif @@ -303,6 +304,20 @@ abort_handler(struct trapframe *tf, int prefetch) idx = FSR_TO_FAULT(fsr); usermode = TRAPF_USERMODE(tf); /* Abort came from user mode? */ + + /* + * Apply BP hardening by flushing the branch prediction cache + * for prefaults on kernel addresses. + */ + if (__predict_false(prefetch && far > VM_MAXUSER_ADDRESS && + (idx == FAULT_TRAN_L2 || idx == FAULT_PERM_L2))) { + bp_harden = PCPU_GET(bp_harden_kind); + if (bp_harden == PCPU_BP_HARDEN_KIND_BPIALL) + _CP15_BPIALL(); + else if (bp_harden == PCPU_BP_HARDEN_KIND_ICIALLU) + _CP15_ICIALLU(); + } + if (usermode) td->td_frame = tf; Modified: head/sys/arm/include/cpuinfo.h ============================================================================== --- head/sys/arm/include/cpuinfo.h Sat Jan 27 09:49:47 2018 (r328466) +++ head/sys/arm/include/cpuinfo.h Sat Jan 27 11:19:41 2018 (r328467) @@ -49,6 +49,7 @@ #define CPU_ARCH_CORTEX_A57 0xD07 #define CPU_ARCH_CORTEX_A72 0xD08 #define CPU_ARCH_CORTEX_A73 0xD09 +#define CPU_ARCH_CORTEX_A75 0xD0A /* QCOM */ @@ -125,6 +126,7 @@ extern struct cpuinfo cpuinfo; void cpuinfo_init(void); #if __ARM_ARCH >= 6 +void cpuinfo_init_bp_hardening(void); void cpuinfo_reinit_mmu(uint32_t ttb); #endif #endif /* _MACHINE_CPUINFO_H_ */ Modified: head/sys/arm/include/pcpu.h ============================================================================== --- head/sys/arm/include/pcpu.h Sat Jan 27 09:49:47 2018 (r328466) +++ head/sys/arm/include/pcpu.h Sat Jan 27 11:19:41 2018 (r328467) @@ -44,6 +44,10 @@ struct vmspace; #endif /* _KERNEL */ #if __ARM_ARCH >= 6 +/* Branch predictor hardening method */ +#define PCPU_BP_HARDEN_KIND_NONE 0 +#define PCPU_BP_HARDEN_KIND_BPIALL 1 +#define PCPU_BP_HARDEN_KIND_ICIALLU 2 #define PCPU_MD_FIELDS \ unsigned int pc_vfpsid; \ @@ -59,7 +63,9 @@ struct vmspace; void *pc_qmap_pte2p; \ unsigned int pc_dbreg[32]; \ int pc_dbreg_cmd; \ - char __pad[155] + int pc_bp_harden_kind; \ + uint32_t pc_original_actlr; \ + char __pad[147] #else #define PCPU_MD_FIELDS \ char __pad[93]
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201801271119.w0RBJfD3027819>