Date: Wed, 22 Jul 2009 15:41:37 +0000 (UTC) From: Andriy Gapon <avg@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-stable@freebsd.org, svn-src-stable-7@freebsd.org Subject: svn commit: r195824 - in stable/7/sys: . cddl/dev/dtrace/amd64 cddl/dev/dtrace/i386 contrib/pf Message-ID: <200907221541.n6MFfbEo045455@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: avg Date: Wed Jul 22 15:41:37 2009 New Revision: 195824 URL: http://svn.freebsd.org/changeset/base/195824 Log: MFC 195710: dtrace_gethrtime: improve scaling of TSC ticks to nanoseconds PR: kern/127441 Modified: stable/7/sys/ (props changed) stable/7/sys/cddl/dev/dtrace/amd64/dtrace_subr.c stable/7/sys/cddl/dev/dtrace/i386/dtrace_subr.c stable/7/sys/contrib/pf/ (props changed) Modified: stable/7/sys/cddl/dev/dtrace/amd64/dtrace_subr.c ============================================================================== --- stable/7/sys/cddl/dev/dtrace/amd64/dtrace_subr.c Wed Jul 22 15:26:19 2009 (r195823) +++ stable/7/sys/cddl/dev/dtrace/amd64/dtrace_subr.c Wed Jul 22 15:41:37 2009 (r195824) @@ -366,6 +366,10 @@ dtrace_safe_defer_signal(void) static int64_t tgt_cpu_tsc; static int64_t hst_cpu_tsc; static int64_t tsc_skew[MAXCPU]; +static uint64_t nsec_scale; + +/* See below for the explanation of this macro. */ +#define SCALE_SHIFT 28 static void dtrace_gethrtime_init_sync(void *arg) @@ -401,9 +405,36 @@ dtrace_gethrtime_init_cpu(void *arg) static void dtrace_gethrtime_init(void *arg) { + uint64_t tsc_f; cpumask_t map; int i; - struct pcpu *cp; + + /* + * Get TSC frequency known at this moment. + * This should be constant if TSC is invariant. + * Otherwise tick->time conversion will be inaccurate, but + * will preserve monotonic property of TSC. + */ + tsc_f = tsc_freq; + + /* + * The following line checks that nsec_scale calculated below + * doesn't overflow 32-bit unsigned integer, so that it can multiply + * another 32-bit integer without overflowing 64-bit. + * Thus minimum supported TSC frequency is 62.5MHz. + */ + KASSERT(tsc_f > (NANOSEC >> (32 - SCALE_SHIFT)), ("TSC frequency is too low")); + + /* + * We scale up NANOSEC/tsc_f ratio to preserve as much precision + * as possible. + * 2^28 factor was chosen quite arbitrarily from practical + * considerations: + * - it supports TSC frequencies as low as 62.5MHz (see above); + * - it provides quite good precision (e < 0.01%) up to THz + * (terahertz) values; + */ + nsec_scale = ((uint64_t)NANOSEC << SCALE_SHIFT) / tsc_f; /* The current CPU is the reference one. */ tsc_skew[curcpu] = 0; @@ -412,7 +443,7 @@ dtrace_gethrtime_init(void *arg) if (i == curcpu) continue; - if ((cp = pcpu_find(i)) == NULL) + if (pcpu_find(i) == NULL) continue; map = 0; @@ -439,7 +470,21 @@ SYSINIT(dtrace_gethrtime_init, SI_SUB_SM uint64_t dtrace_gethrtime() { - return ((rdtsc() + tsc_skew[curcpu]) * (int64_t) 1000000000 / tsc_freq); + uint64_t tsc; + uint32_t lo; + uint32_t hi; + + /* + * We split TSC value into lower and higher 32-bit halves and separately + * scale them with nsec_scale, then we scale them down by 2^28 + * (see nsec_scale calculations) taking into account 32-bit shift of + * the higher half and finally add. + */ + tsc = rdtsc() + tsc_skew[curcpu]; + lo = tsc; + hi = tsc >> 32; + return (((lo * nsec_scale) >> SCALE_SHIFT) + + ((hi * nsec_scale) << (32 - SCALE_SHIFT))); } uint64_t Modified: stable/7/sys/cddl/dev/dtrace/i386/dtrace_subr.c ============================================================================== --- stable/7/sys/cddl/dev/dtrace/i386/dtrace_subr.c Wed Jul 22 15:26:19 2009 (r195823) +++ stable/7/sys/cddl/dev/dtrace/i386/dtrace_subr.c Wed Jul 22 15:41:37 2009 (r195824) @@ -366,6 +366,10 @@ dtrace_safe_defer_signal(void) static int64_t tgt_cpu_tsc; static int64_t hst_cpu_tsc; static int64_t tsc_skew[MAXCPU]; +static uint64_t nsec_scale; + +/* See below for the explanation of this macro. */ +#define SCALE_SHIFT 28 static void dtrace_gethrtime_init_sync(void *arg) @@ -401,10 +405,38 @@ dtrace_gethrtime_init_cpu(void *arg) static void dtrace_gethrtime_init(void *arg) { + uint64_t tsc_f; cpumask_t map; int i; struct pcpu *cp; + /* + * Get TSC frequency known at this moment. + * This should be constant if TSC is invariant. + * Otherwise tick->time conversion will be inaccurate, but + * will preserve monotonic property of TSC. + */ + tsc_f = tsc_freq; + + /* + * The following line checks that nsec_scale calculated below + * doesn't overflow 32-bit unsigned integer, so that it can multiply + * another 32-bit integer without overflowing 64-bit. + * Thus minimum supported TSC frequency is 62.5MHz. + */ + KASSERT(tsc_f > (NANOSEC >> (32 - SCALE_SHIFT)), ("TSC frequency is too low")); + + /* + * We scale up NANOSEC/tsc_f ratio to preserve as much precision + * as possible. + * 2^28 factor was chosen quite arbitrarily from practical + * considerations: + * - it supports TSC frequencies as low as 62.5MHz (see above); + * - it provides quite good precision (e < 0.01%) up to THz + * (terahertz) values; + */ + nsec_scale = ((uint64_t)NANOSEC << SCALE_SHIFT) / tsc_f; + /* The current CPU is the reference one. */ tsc_skew[curcpu] = 0; @@ -439,7 +471,21 @@ SYSINIT(dtrace_gethrtime_init, SI_SUB_SM uint64_t dtrace_gethrtime() { - return ((rdtsc() + tsc_skew[curcpu]) * (int64_t) 1000000000 / tsc_freq); + uint64_t tsc; + uint32_t lo; + uint32_t hi; + + /* + * We split TSC value into lower and higher 32-bit halves and separately + * scale them with nsec_scale, then we scale them down by 2^28 + * (see nsec_scale calculations) taking into account 32-bit shift of + * the higher half and finally add. + */ + tsc = rdtsc() + tsc_skew[curcpu]; + lo = tsc; + hi = tsc >> 32; + return (((lo * nsec_scale) >> SCALE_SHIFT) + + ((hi * nsec_scale) << (32 - SCALE_SHIFT))); } uint64_t
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?200907221541.n6MFfbEo045455>