From owner-dev-commits-src-all@freebsd.org Sun Jan 10 02:43:18 2021 Return-Path: Delivered-To: dev-commits-src-all@mailman.nyi.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2610:1c1:1:606c::19:1]) by mailman.nyi.freebsd.org (Postfix) with ESMTP id 5EDBF4EAF45; Sun, 10 Jan 2021 02:43:18 +0000 (UTC) (envelope-from git@FreeBSD.org) Received: from mxrelay.nyi.freebsd.org (mxrelay.nyi.freebsd.org [IPv6:2610:1c1:1:606c::19:3]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits) server-digest SHA256 client-signature RSA-PSS (4096 bits) client-digest SHA256) (Client CN "mxrelay.nyi.freebsd.org", Issuer "Let's Encrypt Authority X3" (verified OK)) by mx1.freebsd.org (Postfix) with ESMTPS id 4DD1Nh5n2tz4qWS; Sun, 10 Jan 2021 02:43:16 +0000 (UTC) (envelope-from git@FreeBSD.org) Received: from gitrepo.freebsd.org (gitrepo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:5]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits) server-digest SHA256) (Client did not present a certificate) by mxrelay.nyi.freebsd.org (Postfix) with ESMTPS id E907720673; Sun, 10 Jan 2021 02:43:15 +0000 (UTC) (envelope-from git@FreeBSD.org) Received: from gitrepo.freebsd.org ([127.0.1.44]) by gitrepo.freebsd.org (8.16.1/8.16.1) with ESMTP id 10A2hF0S058127; Sun, 10 Jan 2021 02:43:15 GMT (envelope-from git@gitrepo.freebsd.org) Received: (from git@localhost) by gitrepo.freebsd.org (8.16.1/8.16.1/Submit) id 10A2hFWG058126; Sun, 10 Jan 2021 02:43:15 GMT (envelope-from git) Date: Sun, 10 Jan 2021 02:43:15 GMT Message-Id: <202101100243.10A2hFWG058126@gitrepo.freebsd.org> To: src-committers@FreeBSD.org, dev-commits-src-all@FreeBSD.org, dev-commits-src-main@FreeBSD.org From: Konstantin Belousov Subject: git: 5bf4bafd13a4 - main - x86 vdso gettc: eliminate duplicated code in ifunc selectors. MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit X-Git-Committer: kib X-Git-Repository: src X-Git-Refname: refs/heads/main X-Git-Reftype: branch X-Git-Commit: 5bf4bafd13a4044b7c3d2e8246684a597c6f2134 Auto-Submitted: auto-generated X-BeenThere: dev-commits-src-all@freebsd.org X-Mailman-Version: 2.1.34 Precedence: list List-Id: Commit messages for all branches of the src repository List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Sun, 10 Jan 2021 02:43:18 -0000 The branch main has been updated by kib: URL: https://cgit.FreeBSD.org/src/commit/?id=5bf4bafd13a4044b7c3d2e8246684a597c6f2134 commit 5bf4bafd13a4044b7c3d2e8246684a597c6f2134 Author: Konstantin Belousov AuthorDate: 2021-01-06 15:00:48 +0000 Commit: Konstantin Belousov CommitDate: 2021-01-10 02:42:34 +0000 x86 vdso gettc: eliminate duplicated code in ifunc selectors. Create array of rdtsc selectors and provide helper that calculate the index into the selectors array. Reviewed by: gallatin, markj MFC after: 1 week Sponsored by: The FreeBSD Foundation Differential Revision: https://reviews.freebsd.org/D27986 --- lib/libc/x86/sys/__vdso_gettc.c | 87 ++++++++++++++++++++++++++++++----------- 1 file changed, 65 insertions(+), 22 deletions(-) diff --git a/lib/libc/x86/sys/__vdso_gettc.c b/lib/libc/x86/sys/__vdso_gettc.c index b08b11e9c714..4e0fc9f7e841 100644 --- a/lib/libc/x86/sys/__vdso_gettc.c +++ b/lib/libc/x86/sys/__vdso_gettc.c @@ -83,20 +83,6 @@ rdtsc_low_mb_none(const struct vdso_timehands *th) return (rdtsc_low(th)); } -DEFINE_UIFUNC(static, u_int, __vdso_gettc_rdtsc_low, - (const struct vdso_timehands *th)) -{ - u_int p[4]; - /* Not a typo, string matches our do_cpuid() registers use. */ - static const char intel_id[] = "GenuntelineI"; - - if ((cpu_feature & CPUID_SSE2) == 0) - return (rdtsc_low_mb_none); - do_cpuid(0, p); - return (memcmp(p + 1, intel_id, sizeof(intel_id) - 1) == 0 ? - rdtsc_low_mb_lfence : rdtsc_low_mb_mfence); -} - static u_int rdtsc32_mb_lfence(void) { @@ -117,17 +103,74 @@ rdtsc32_mb_none(void) return (rdtsc32()); } -DEFINE_UIFUNC(static, u_int, __vdso_gettc_rdtsc32, (void)) +struct tsc_selector_tag { + u_int (*ts_rdtsc32)(void); + u_int (*ts_rdtsc_low)(const struct vdso_timehands *); +}; + +static const struct tsc_selector_tag tsc_selector[] = { + [0] = { /* Intel or AMD Zen+, LFENCE */ + .ts_rdtsc32 = rdtsc32_mb_lfence, + .ts_rdtsc_low = rdtsc_low_mb_lfence, + }, + [1] = { /* AMD, MFENCE */ + .ts_rdtsc32 = rdtsc32_mb_mfence, + .ts_rdtsc_low = rdtsc_low_mb_mfence, + }, + [2] = { /* No SSE2 */ + .ts_rdtsc32 = rdtsc32_mb_none, + .ts_rdtsc_low = rdtsc_low_mb_none, + }, +}; + +static int +tsc_selector_idx(u_int cpu_feature) { - u_int p[4]; - /* Not a typo, string matches our do_cpuid() registers use. */ - static const char intel_id[] = "GenuntelineI"; + u_int amd_feature, cpu_exthigh, cpu_id, p[4], v[3]; + static const char amd_id[] = "AuthenticAMD"; + static const char hygon_id[] = "HygonGenuine"; + bool amd_cpu; + + if (cpu_feature == 0) + return (2); /* should not happen due to RDTSC */ - if ((cpu_feature & CPUID_SSE2) == 0) - return (rdtsc32_mb_none); do_cpuid(0, p); - return (memcmp(p + 1, intel_id, sizeof(intel_id) - 1) == 0 ? - rdtsc32_mb_lfence : rdtsc32_mb_mfence); + v[0] = p[1]; + v[1] = p[3]; + v[2] = p[2]; + amd_cpu = memcmp(v, amd_id, sizeof(amd_id) - 1) == 0 || + memcmp(v, hygon_id, sizeof(hygon_id) - 1) == 0; + + do_cpuid(1, p); + cpu_id = p[0]; + + if (cpu_feature != 0) { + do_cpuid(0x80000000, p); + cpu_exthigh = p[0]; + } else { + cpu_exthigh = 0; + } + if (cpu_exthigh >= 0x80000001) { + do_cpuid(0x80000001, p); + amd_feature = p[3]; + } else { + amd_feature = 0; + } + + if ((cpu_feature & CPUID_SSE2) == 0) + return (2); + return (amd_cpu ? 1 : 0); +} + +DEFINE_UIFUNC(static, u_int, __vdso_gettc_rdtsc_low, + (const struct vdso_timehands *th)) +{ + return (tsc_selector[tsc_selector_idx(cpu_feature)].ts_rdtsc_low); +} + +DEFINE_UIFUNC(static, u_int, __vdso_gettc_rdtsc32, (void)) +{ + return (tsc_selector[tsc_selector_idx(cpu_feature)].ts_rdtsc32); } #define HPET_DEV_MAP_MAX 10