From owner-svn-src-all@freebsd.org Mon Nov 18 20:53:58 2019 Return-Path: Delivered-To: svn-src-all@mailman.nyi.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2610:1c1:1:606c::19:1]) by mailman.nyi.freebsd.org (Postfix) with ESMTP id B10931C97B8; Mon, 18 Nov 2019 20:53:58 +0000 (UTC) (envelope-from kib@FreeBSD.org) Received: from mxrelay.nyi.freebsd.org (mxrelay.nyi.freebsd.org [IPv6:2610:1c1:1:606c::19:3]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) server-signature RSA-PSS (4096 bits) client-signature RSA-PSS (4096 bits) client-digest SHA256) (Client CN "mxrelay.nyi.freebsd.org", Issuer "Let's Encrypt Authority X3" (verified OK)) by mx1.freebsd.org (Postfix) with ESMTPS id 47H1QZ3nPBz4Wsx; Mon, 18 Nov 2019 20:53:58 +0000 (UTC) (envelope-from kib@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mxrelay.nyi.freebsd.org (Postfix) with ESMTPS id 651052F9BE; Mon, 18 Nov 2019 20:53:58 +0000 (UTC) (envelope-from kib@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id xAIKrwPJ032718; Mon, 18 Nov 2019 20:53:58 GMT (envelope-from kib@FreeBSD.org) Received: (from kib@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id xAIKrv4Z032716; Mon, 18 Nov 2019 20:53:57 GMT (envelope-from kib@FreeBSD.org) Message-Id: <201911182053.xAIKrv4Z032716@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: kib set sender to kib@FreeBSD.org using -f From: Konstantin Belousov Date: Mon, 18 Nov 2019 20:53:57 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r354828 - in head/sys: dev/hwpmc x86/include x86/x86 X-SVN-Group: head X-SVN-Commit-Author: kib X-SVN-Commit-Paths: in head/sys: dev/hwpmc x86/include x86/x86 X-SVN-Commit-Revision: 354828 X-SVN-Commit-Repository: base MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-all@freebsd.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: "SVN commit messages for the entire src tree \(except for " user" and " projects" \)" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Mon, 18 Nov 2019 20:53:58 -0000 Author: kib Date: Mon Nov 18 20:53:57 2019 New Revision: 354828 URL: https://svnweb.freebsd.org/changeset/base/354828 Log: Add x86 msr tweak KPI. Use the KPI to tweak MSRs in mitigation code. Reviewed by: markj, scottl Sponsored by: The FreeBSD Foundation MFC after: 1 week Differential revision: https://reviews.freebsd.org/D22431 Modified: head/sys/dev/hwpmc/hwpmc_core.c head/sys/x86/include/x86_var.h head/sys/x86/x86/cpu_machdep.c Modified: head/sys/dev/hwpmc/hwpmc_core.c ============================================================================== --- head/sys/dev/hwpmc/hwpmc_core.c Mon Nov 18 20:07:43 2019 (r354827) +++ head/sys/dev/hwpmc/hwpmc_core.c Mon Nov 18 20:53:57 2019 (r354828) @@ -220,15 +220,6 @@ iaf_reload_count_to_perfctr_value(pmc_value_t rlc) return (1ULL << core_iaf_width) - rlc; } -static void -tweak_tsx_force_abort(void *arg) -{ - u_int val; - - val = (uintptr_t)arg; - wrmsr(MSR_TSX_FORCE_ABORT, val); -} - static int iaf_allocate_pmc(int cpu, int ri, struct pmc *pm, const struct pmc_op_pmcallocate *a) @@ -270,7 +261,8 @@ iaf_allocate_pmc(int cpu, int ri, struct pmc *pm, if ((cpu_stdext_feature3 & CPUID_STDEXT3_TSXFA) != 0 && !pmc_tsx_force_abort_set) { pmc_tsx_force_abort_set = true; - smp_rendezvous(NULL, tweak_tsx_force_abort, NULL, (void *)1); + x86_msr_op(MSR_TSX_FORCE_ABORT, MSR_OP_RENDEZVOUS | + MSR_OP_WRITE, 1); } flags = 0; @@ -411,7 +403,8 @@ iaf_release_pmc(int cpu, int ri, struct pmc *pmc) MPASS(pmc_alloc_refs > 0); if (pmc_alloc_refs-- == 1 && pmc_tsx_force_abort_set) { pmc_tsx_force_abort_set = false; - smp_rendezvous(NULL, tweak_tsx_force_abort, NULL, (void *)0); + x86_msr_op(MSR_TSX_FORCE_ABORT, MSR_OP_RENDEZVOUS | + MSR_OP_WRITE, 0); } return (0); Modified: head/sys/x86/include/x86_var.h ============================================================================== --- head/sys/x86/include/x86_var.h Mon Nov 18 20:07:43 2019 (r354827) +++ head/sys/x86/include/x86_var.h Mon Nov 18 20:53:57 2019 (r354828) @@ -148,4 +148,12 @@ int user_dbreg_trap(register_t dr6); int minidumpsys(struct dumperinfo *); struct pcb *get_pcb_td(struct thread *td); +#define MSR_OP_ANDNOT 0x00000001 +#define MSR_OP_OR 0x00000002 +#define MSR_OP_WRITE 0x00000003 +#define MSR_OP_LOCAL 0x10000000 +#define MSR_OP_SCHED 0x20000000 +#define MSR_OP_RENDEZVOUS 0x30000000 +void x86_msr_op(u_int msr, u_int op, uint64_t arg1); + #endif Modified: head/sys/x86/x86/cpu_machdep.c ============================================================================== --- head/sys/x86/x86/cpu_machdep.c Mon Nov 18 20:07:43 2019 (r354827) +++ head/sys/x86/x86/cpu_machdep.c Mon Nov 18 20:53:57 2019 (r354828) @@ -111,6 +111,80 @@ static u_int cpu_reset_proxyid; static volatile u_int cpu_reset_proxy_active; #endif +struct msr_op_arg { + u_int msr; + int op; + uint64_t arg1; +}; + +static void +x86_msr_op_one(void *argp) +{ + struct msr_op_arg *a; + uint64_t v; + + a = argp; + switch (a->op) { + case MSR_OP_ANDNOT: + v = rdmsr(a->msr); + v &= ~a->arg1; + wrmsr(a->msr, v); + break; + case MSR_OP_OR: + v = rdmsr(a->msr); + v |= a->arg1; + wrmsr(a->msr, v); + break; + case MSR_OP_WRITE: + wrmsr(a->msr, a->arg1); + break; + } +} + +#define MSR_OP_EXMODE_MASK 0xf0000000 +#define MSR_OP_OP_MASK 0x000000ff + +void +x86_msr_op(u_int msr, u_int op, uint64_t arg1) +{ + struct thread *td; + struct msr_op_arg a; + u_int exmode; + int bound_cpu, i, is_bound; + + a.op = op & MSR_OP_OP_MASK; + MPASS(a.op == MSR_OP_ANDNOT || a.op == MSR_OP_OR || + a.op == MSR_OP_WRITE); + exmode = op & MSR_OP_EXMODE_MASK; + MPASS(exmode == MSR_OP_LOCAL || exmode == MSR_OP_SCHED || + exmode == MSR_OP_RENDEZVOUS); + a.msr = msr; + a.arg1 = arg1; + switch (exmode) { + case MSR_OP_LOCAL: + x86_msr_op_one(&a); + break; + case MSR_OP_SCHED: + td = curthread; + thread_lock(td); + is_bound = sched_is_bound(td); + bound_cpu = td->td_oncpu; + CPU_FOREACH(i) { + sched_bind(td, i); + x86_msr_op_one(&a); + } + if (is_bound) + sched_bind(td, bound_cpu); + else + sched_unbind(td); + thread_unlock(td); + break; + case MSR_OP_RENDEZVOUS: + smp_rendezvous(NULL, x86_msr_op_one, NULL, &a); + break; + } +} + /* * Automatically initialized per CPU errata in cpu_idle_tun below. */ @@ -806,18 +880,10 @@ SYSCTL_INT(_hw, OID_AUTO, ibrs_active, CTLFLAG_RD, &hw void hw_ibrs_recalculate(void) { - uint64_t v; - if ((cpu_ia32_arch_caps & IA32_ARCH_CAP_IBRS_ALL) != 0) { - if (hw_ibrs_disable) { - v = rdmsr(MSR_IA32_SPEC_CTRL); - v &= ~(uint64_t)IA32_SPEC_CTRL_IBRS; - wrmsr(MSR_IA32_SPEC_CTRL, v); - } else { - v = rdmsr(MSR_IA32_SPEC_CTRL); - v |= IA32_SPEC_CTRL_IBRS; - wrmsr(MSR_IA32_SPEC_CTRL, v); - } + x86_msr_op(MSR_IA32_SPEC_CTRL, MSR_OP_LOCAL | + (hw_ibrs_disable ? MSR_OP_ANDNOT : MSR_OP_OR), + IA32_SPEC_CTRL_IBRS); return; } hw_ibrs_active = (cpu_stdext_feature3 & CPUID_STDEXT3_IBPB) != 0 && @@ -849,46 +915,17 @@ SYSCTL_INT(_hw, OID_AUTO, spec_store_bypass_disable_ac "Speculative Store Bypass Disable active"); static void -hw_ssb_set_one(bool enable) -{ - uint64_t v; - - v = rdmsr(MSR_IA32_SPEC_CTRL); - if (enable) - v |= (uint64_t)IA32_SPEC_CTRL_SSBD; - else - v &= ~(uint64_t)IA32_SPEC_CTRL_SSBD; - wrmsr(MSR_IA32_SPEC_CTRL, v); -} - -static void hw_ssb_set(bool enable, bool for_all_cpus) { - struct thread *td; - int bound_cpu, i, is_bound; if ((cpu_stdext_feature3 & CPUID_STDEXT3_SSBD) == 0) { hw_ssb_active = 0; return; } hw_ssb_active = enable; - if (for_all_cpus) { - td = curthread; - thread_lock(td); - is_bound = sched_is_bound(td); - bound_cpu = td->td_oncpu; - CPU_FOREACH(i) { - sched_bind(td, i); - hw_ssb_set_one(enable); - } - if (is_bound) - sched_bind(td, bound_cpu); - else - sched_unbind(td); - thread_unlock(td); - } else { - hw_ssb_set_one(enable); - } + x86_msr_op(MSR_IA32_SPEC_CTRL, + (enable ? MSR_OP_OR : MSR_OP_ANDNOT) | + (for_all_cpus ? MSR_OP_SCHED : MSR_OP_LOCAL), IA32_SPEC_CTRL_SSBD); } void @@ -1151,43 +1188,13 @@ enum { }; static void -taa_set_one(bool enable) -{ - uint64_t v; - - v = rdmsr(MSR_IA32_TSX_CTRL); - if (enable) - v |= (uint64_t)(IA32_TSX_CTRL_RTM_DISABLE | - IA32_TSX_CTRL_TSX_CPUID_CLEAR); - else - v &= ~(uint64_t)(IA32_TSX_CTRL_RTM_DISABLE | - IA32_TSX_CTRL_TSX_CPUID_CLEAR); - - wrmsr(MSR_IA32_TSX_CTRL, v); -} - -static void taa_set(bool enable, bool all) { - struct thread *td; - int bound_cpu, i, is_bound; - if (all) { - td = curthread; - thread_lock(td); - is_bound = sched_is_bound(td); - bound_cpu = td->td_oncpu; - CPU_FOREACH(i) { - sched_bind(td, i); - taa_set_one(enable); - } - if (is_bound) - sched_bind(td, bound_cpu); - else - sched_unbind(td); - thread_unlock(td); - } else - taa_set_one(enable); + x86_msr_op(MSR_IA32_TSX_CTRL, + (enable ? MSR_OP_OR : MSR_OP_ANDNOT) | + (all ? MSR_OP_RENDEZVOUS : MSR_OP_LOCAL), + IA32_TSX_CTRL_RTM_DISABLE | IA32_TSX_CTRL_TSX_CPUID_CLEAR); } void