Date: Sun, 10 Jan 2021 02:43:13 GMT From: Konstantin Belousov <kib@FreeBSD.org> To: src-committers@FreeBSD.org, dev-commits-src-all@FreeBSD.org, dev-commits-src-main@FreeBSD.org Subject: git: 9e680e4005b7 - main - tsc: add RDTSCP or faster variants of get_timecount() Message-ID: <202101100243.10A2hDBv058086@gitrepo.freebsd.org>
next in thread | raw e-mail | index | archive | help
The branch main has been updated by kib: URL: https://cgit.FreeBSD.org/src/commit/?id=9e680e4005b77e3028d28377ee3722a5260f4422 commit 9e680e4005b77e3028d28377ee3722a5260f4422 Author: Konstantin Belousov <kib@FreeBSD.org> AuthorDate: 2021-01-05 21:00:14 +0000 Commit: Konstantin Belousov <kib@FreeBSD.org> CommitDate: 2021-01-10 02:42:34 +0000 tsc: add RDTSCP or faster variants of get_timecount() Use it in preference of Xfenced RDTSC if RDTSCP is supported. It is recommended by both Intel and AMD. But, on AMD Zens and newer use LFENCE, as recommended by AMD [*]. In particular, this means that now AMD CPUs use more appropriate fence instead of too harsh MFENCe. Add comment explaining the intent of the selection logic. Reported by: gallatin [*] Reviewed by: gallatin, markj Tested by: gallatin, pho MFC after: 1 week Sponsored by: The FreeBSD Foundation Differential Revision: https://reviews.freebsd.org/D27986 --- sys/x86/x86/tsc.c | 39 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/sys/x86/x86/tsc.c b/sys/x86/x86/tsc.c index 8b6a420ae1e4..ad8108c8a7ea 100644 --- a/sys/x86/x86/tsc.c +++ b/sys/x86/x86/tsc.c @@ -97,6 +97,8 @@ static u_int tsc_get_timecount_lfence(struct timecounter *tc); static u_int tsc_get_timecount_low_lfence(struct timecounter *tc); static u_int tsc_get_timecount_mfence(struct timecounter *tc); static u_int tsc_get_timecount_low_mfence(struct timecounter *tc); +static u_int tscp_get_timecount(struct timecounter *tc); +static u_int tscp_get_timecount_low(struct timecounter *tc); static void tsc_levels_changed(void *arg, int unit); static uint32_t x86_tsc_vdso_timehands(struct vdso_timehands *vdso_th, struct timecounter *tc); @@ -628,7 +630,25 @@ init_TSC_tc(void) init: for (shift = 0; shift <= 31 && (tsc_freq >> shift) > max_freq; shift++) ; - if ((cpu_feature & CPUID_SSE2) != 0 && mp_ncpus > 1) { + + /* + * Timecounter implementation selection, top to bottom: + * - For AMD Zens and newer, use LFENCE;RDTSC. + * - If RDTSCP is available, use RDTSCP. + * - If fence instructions are provided (SSE2), use LFENCE;RDTSC + * on Intel, and MFENCE;RDTSC on AMD. + * - For really old CPUs, just use RDTSC. + */ + if ((cpu_vendor_id == CPU_VENDOR_AMD || + cpu_vendor_id == CPU_VENDOR_HYGON) && + CPUID_TO_FAMILY(cpu_id) >= 0x17) { + tsc_timecounter.tc_get_timecount = shift > 0 ? + tsc_get_timecount_low_lfence : + tsc_get_timecount_lfence; + } else if ((amd_feature & AMDID_RDTSCP) != 0) { + tsc_timecounter.tc_get_timecount = shift > 0 ? + tscp_get_timecount_low : tscp_get_timecount; + } else if ((cpu_feature & CPUID_SSE2) != 0 && mp_ncpus > 1) { if (cpu_vendor_id == CPU_VENDOR_AMD || cpu_vendor_id == CPU_VENDOR_HYGON) { tsc_timecounter.tc_get_timecount = shift > 0 ? @@ -783,6 +803,13 @@ tsc_get_timecount(struct timecounter *tc __unused) return (rdtsc32()); } +static u_int +tscp_get_timecount(struct timecounter *tc __unused) +{ + + return (rdtscp32()); +} + static inline u_int tsc_get_timecount_low(struct timecounter *tc) { @@ -793,6 +820,16 @@ tsc_get_timecount_low(struct timecounter *tc) return (rv); } +static u_int +tscp_get_timecount_low(struct timecounter *tc) +{ + uint32_t rv; + + __asm __volatile("rdtscp; movl %1, %%ecx; shrd %%cl, %%edx, %0" + : "=a" (rv) : "m" (tc->tc_priv) : "ecx", "edx"); + return (rv); +} + static u_int tsc_get_timecount_lfence(struct timecounter *tc __unused) {
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?202101100243.10A2hDBv058086>