Date: Thu, 7 Apr 2011 23:28:28 +0000 (UTC) From: Jung-uk Kim <jkim@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r220433 - in head/sys: amd64/amd64 cddl/dev/dtrace/amd64 cddl/dev/dtrace/i386 compat/linprocfs contrib/altq/altq dev/acpica i386/i386 i386/isa pc98/pc98 x86/cpufreq x86/isa x86/x86 Message-ID: <201104072328.p37NSSsO055101@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: jkim Date: Thu Apr 7 23:28:28 2011 New Revision: 220433 URL: http://svn.freebsd.org/changeset/base/220433 Log: Use atomic load & store for TSC frequency. It may be overkill for amd64 but safer for i386 because it can be easily over 4 GHz now. More worse, it can be easily changed by user with 'machdep.tsc_freq' tunable (directly) or cpufreq(4) (indirectly). Note it is intentionally not used in performance critical paths to avoid performance regression (but we should, in theory). Alternatively, we may add "virtual TSC" with lower frequency if maximum frequency overflows 32 bits (and ignore possible incoherency as we do now). Modified: head/sys/amd64/amd64/legacy.c head/sys/amd64/amd64/machdep.c head/sys/amd64/amd64/prof_machdep.c head/sys/cddl/dev/dtrace/amd64/dtrace_subr.c head/sys/cddl/dev/dtrace/i386/dtrace_subr.c head/sys/compat/linprocfs/linprocfs.c head/sys/contrib/altq/altq/altq_subr.c head/sys/dev/acpica/acpi_cpu.c head/sys/i386/i386/legacy.c head/sys/i386/i386/machdep.c head/sys/i386/i386/perfmon.c head/sys/i386/isa/prof_machdep.c head/sys/pc98/pc98/machdep.c head/sys/x86/cpufreq/est.c head/sys/x86/isa/clock.c head/sys/x86/x86/tsc.c Modified: head/sys/amd64/amd64/legacy.c ============================================================================== --- head/sys/amd64/amd64/legacy.c Thu Apr 7 23:14:33 2011 (r220432) +++ head/sys/amd64/amd64/legacy.c Thu Apr 7 23:28:28 2011 (r220433) @@ -321,7 +321,8 @@ cpu_read_ivar(device_t dev, device_t chi break; case CPU_IVAR_NOMINAL_MHZ: if (tsc_is_invariant) { - *result = (uintptr_t)(tsc_freq / 1000000); + *result = (uintptr_t)(atomic_load_acq_64(&tsc_freq) / + 1000000); break; } /* FALLTHROUGH */ Modified: head/sys/amd64/amd64/machdep.c ============================================================================== --- head/sys/amd64/amd64/machdep.c Thu Apr 7 23:14:33 2011 (r220432) +++ head/sys/amd64/amd64/machdep.c Thu Apr 7 23:28:28 2011 (r220433) @@ -546,18 +546,19 @@ int cpu_est_clockrate(int cpu_id, uint64_t *rate) { register_t reg; - uint64_t tsc1, tsc2; + uint64_t freq, tsc1, tsc2; if (pcpu_find(cpu_id) == NULL || rate == NULL) return (EINVAL); + freq = atomic_load_acq_64(&tsc_freq); /* If TSC is P-state invariant, DELAY(9) based logic fails. */ - if (tsc_is_invariant && tsc_freq != 0) + if (tsc_is_invariant && freq != 0) return (EOPNOTSUPP); /* If we're booting, trust the rate calibrated moments ago. */ - if (cold && tsc_freq != 0) { - *rate = tsc_freq; + if (cold && freq != 0) { + *rate = freq; return (0); } @@ -586,7 +587,7 @@ cpu_est_clockrate(int cpu_id, uint64_t * #endif tsc2 -= tsc1; - if (tsc_freq != 0) { + if (freq != 0) { *rate = tsc2 * 1000; return (0); } Modified: head/sys/amd64/amd64/prof_machdep.c ============================================================================== --- head/sys/amd64/amd64/prof_machdep.c Thu Apr 7 23:14:33 2011 (r220432) +++ head/sys/amd64/amd64/prof_machdep.c Thu Apr 7 23:28:28 2011 (r220433) @@ -311,18 +311,22 @@ void startguprof(gp) struct gmonparam *gp; { + uint64_t freq; + + freq = atomic_load_acq_64(&tsc_freq); if (cputime_clock == CPUTIME_CLOCK_UNINITIALIZED) { - cputime_clock = CPUTIME_CLOCK_I8254; - if (tsc_freq != 0 && mp_ncpus == 1) + if (freq != 0 && mp_ncpus == 1) cputime_clock = CPUTIME_CLOCK_TSC; + else + cputime_clock = CPUTIME_CLOCK_I8254; } - gp->profrate = i8254_freq << CPUTIME_CLOCK_I8254_SHIFT; if (cputime_clock == CPUTIME_CLOCK_TSC) { - gp->profrate = tsc_freq >> 1; + gp->profrate = freq >> 1; cputime_prof_active = 1; - } + } else + gp->profrate = i8254_freq << CPUTIME_CLOCK_I8254_SHIFT; #if defined(PERFMON) && defined(I586_PMC_GUPROF) - else if (cputime_clock == CPUTIME_CLOCK_I586_PMC) { + if (cputime_clock == CPUTIME_CLOCK_I586_PMC) { if (perfmon_avail() && perfmon_setup(0, cputime_clock_pmc_conf) == 0) { if (perfmon_start(0) != 0) Modified: head/sys/cddl/dev/dtrace/amd64/dtrace_subr.c ============================================================================== --- head/sys/cddl/dev/dtrace/amd64/dtrace_subr.c Thu Apr 7 23:14:33 2011 (r220432) +++ head/sys/cddl/dev/dtrace/amd64/dtrace_subr.c Thu Apr 7 23:28:28 2011 (r220433) @@ -403,7 +403,7 @@ dtrace_gethrtime_init(void *arg) * Otherwise tick->time conversion will be inaccurate, but * will preserve monotonic property of TSC. */ - tsc_f = tsc_freq; + tsc_f = atomic_load_acq_64(&tsc_freq); /* * The following line checks that nsec_scale calculated below Modified: head/sys/cddl/dev/dtrace/i386/dtrace_subr.c ============================================================================== --- head/sys/cddl/dev/dtrace/i386/dtrace_subr.c Thu Apr 7 23:14:33 2011 (r220432) +++ head/sys/cddl/dev/dtrace/i386/dtrace_subr.c Thu Apr 7 23:28:28 2011 (r220433) @@ -403,7 +403,7 @@ dtrace_gethrtime_init(void *arg) * Otherwise tick->time conversion will be inaccurate, but * will preserve monotonic property of TSC. */ - tsc_f = tsc_freq; + tsc_f = atomic_load_acq_64(&tsc_freq); /* * The following line checks that nsec_scale calculated below Modified: head/sys/compat/linprocfs/linprocfs.c ============================================================================== --- head/sys/compat/linprocfs/linprocfs.c Thu Apr 7 23:14:33 2011 (r220432) +++ head/sys/compat/linprocfs/linprocfs.c Thu Apr 7 23:28:28 2011 (r220433) @@ -221,6 +221,7 @@ linprocfs_docpuinfo(PFS_FILL_ARGS) { int hw_model[2]; char model[128]; + uint64_t freq; size_t size; int class, fqmhz, fqkhz; int i; @@ -303,9 +304,10 @@ linprocfs_docpuinfo(PFS_FILL_ARGS) if (cpu_feature & (1 << i)) sbuf_printf(sb, " %s", flags[i]); sbuf_cat(sb, "\n"); - if (class >= 5) { - fqmhz = (tsc_freq + 4999) / 1000000; - fqkhz = ((tsc_freq + 4999) / 10000) % 100; + freq = atomic_load_acq_64(&tsc_freq); + if (freq != 0) { + fqmhz = (freq + 4999) / 1000000; + fqkhz = ((freq + 4999) / 10000) % 100; sbuf_printf(sb, "cpu MHz\t\t: %d.%02d\n" "bogomips\t: %d.%02d\n", Modified: head/sys/contrib/altq/altq/altq_subr.c ============================================================================== --- head/sys/contrib/altq/altq/altq_subr.c Thu Apr 7 23:14:33 2011 (r220432) +++ head/sys/contrib/altq/altq/altq_subr.c Thu Apr 7 23:28:28 2011 (r220433) @@ -929,7 +929,8 @@ init_machclk_setup(void) #if defined(__amd64__) || defined(__i386__) /* check if TSC is available */ #ifdef __FreeBSD__ - if ((cpu_feature & CPUID_TSC) == 0 || tsc_freq == 0) + if ((cpu_feature & CPUID_TSC) == 0 || + atomic_load_acq_64(&tsc_freq) == 0) #else if ((cpu_feature & CPUID_TSC) == 0) #endif @@ -964,7 +965,7 @@ init_machclk(void) */ #if defined(__amd64__) || defined(__i386__) #ifdef __FreeBSD__ - machclk_freq = tsc_freq; + machclk_freq = atomic_load_acq_64(&tsc_freq); #elif defined(__NetBSD__) machclk_freq = (u_int32_t)cpu_tsc_freq; #elif defined(__OpenBSD__) && (defined(I586_CPU) || defined(I686_CPU)) Modified: head/sys/dev/acpica/acpi_cpu.c ============================================================================== --- head/sys/dev/acpica/acpi_cpu.c Thu Apr 7 23:14:33 2011 (r220432) +++ head/sys/dev/acpica/acpi_cpu.c Thu Apr 7 23:28:28 2011 (r220433) @@ -516,7 +516,7 @@ acpi_cpu_read_ivar(device_t dev, device_ #if defined(__amd64__) || defined(__i386__) case CPU_IVAR_NOMINAL_MHZ: if (tsc_is_invariant) { - *result = (uintptr_t)(tsc_freq / 1000000); + *result = (uintptr_t)(atomic_load_acq_64(&tsc_freq) / 1000000); break; } /* FALLTHROUGH */ Modified: head/sys/i386/i386/legacy.c ============================================================================== --- head/sys/i386/i386/legacy.c Thu Apr 7 23:14:33 2011 (r220432) +++ head/sys/i386/i386/legacy.c Thu Apr 7 23:28:28 2011 (r220433) @@ -342,7 +342,8 @@ cpu_read_ivar(device_t dev, device_t chi break; case CPU_IVAR_NOMINAL_MHZ: if (tsc_is_invariant) { - *result = (uintptr_t)(tsc_freq / 1000000); + *result = (uintptr_t)(atomic_load_acq_64(&tsc_freq) / + 1000000); break; } /* FALLTHROUGH */ Modified: head/sys/i386/i386/machdep.c ============================================================================== --- head/sys/i386/i386/machdep.c Thu Apr 7 23:14:33 2011 (r220432) +++ head/sys/i386/i386/machdep.c Thu Apr 7 23:28:28 2011 (r220433) @@ -1137,20 +1137,21 @@ int cpu_est_clockrate(int cpu_id, uint64_t *rate) { register_t reg; - uint64_t tsc1, tsc2; + uint64_t freq, tsc1, tsc2; if (pcpu_find(cpu_id) == NULL || rate == NULL) return (EINVAL); if ((cpu_feature & CPUID_TSC) == 0) return (EOPNOTSUPP); + freq = atomic_load_acq_64(&tsc_freq); /* If TSC is P-state invariant, DELAY(9) based logic fails. */ - if (tsc_is_invariant && tsc_freq != 0) + if (tsc_is_invariant && freq != 0) return (EOPNOTSUPP); /* If we're booting, trust the rate calibrated moments ago. */ - if (cold && tsc_freq != 0) { - *rate = tsc_freq; + if (cold && freq != 0) { + *rate = freq; return (0); } @@ -1179,7 +1180,7 @@ cpu_est_clockrate(int cpu_id, uint64_t * #endif tsc2 -= tsc1; - if (tsc_freq != 0) { + if (freq != 0) { *rate = tsc2 * 1000; return (0); } Modified: head/sys/i386/i386/perfmon.c ============================================================================== --- head/sys/i386/i386/perfmon.c Thu Apr 7 23:14:33 2011 (r220432) +++ head/sys/i386/i386/perfmon.c Thu Apr 7 23:28:28 2011 (r220433) @@ -336,6 +336,7 @@ perfmon_ioctl(struct cdev *dev, u_long c struct pmc *pmc; struct pmc_data *pmcd; struct pmc_tstamp *pmct; + uint64_t freq; int *ip; int rv; @@ -386,13 +387,14 @@ perfmon_ioctl(struct cdev *dev, u_long c break; case PMIOTSTAMP: - if (!tsc_freq) { + freq = atomic_load_acq_64(&tsc_freq); + if (freq == 0) { rv = ENOTTY; break; } pmct = (struct pmc_tstamp *)param; /* XXX interface loses precision. */ - pmct->pmct_rate = tsc_freq / 1000000; + pmct->pmct_rate = freq / 1000000; pmct->pmct_value = rdtsc(); rv = 0; break; Modified: head/sys/i386/isa/prof_machdep.c ============================================================================== --- head/sys/i386/isa/prof_machdep.c Thu Apr 7 23:14:33 2011 (r220432) +++ head/sys/i386/isa/prof_machdep.c Thu Apr 7 23:28:28 2011 (r220433) @@ -170,8 +170,8 @@ cputime() { u_int count; int delta; -#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP) && \ - defined(PERFMON) && defined(I586_PMC_GUPROF) +#if (defined(I586_CPU) || defined(I686_CPU)) && \ + defined(PERFMON) && defined(I586_PMC_GUPROF) && !defined(SMP) u_quad_t event_count; #endif u_char high, low; @@ -286,21 +286,23 @@ void startguprof(gp) struct gmonparam *gp; { - if (cputime_clock == CPUTIME_CLOCK_UNINITIALIZED) { - cputime_clock = CPUTIME_CLOCK_I8254; #if defined(I586_CPU) || defined(I686_CPU) - if (tsc_freq != 0 && mp_ncpus == 1) + uint64_t freq; + + freq = atomic_load_acq_64(&tsc_freq); + if (cputime_clock == CPUTIME_CLOCK_UNINITIALIZED) { + if (freq != 0 && mp_ncpus == 1) cputime_clock = CPUTIME_CLOCK_TSC; -#endif + else + cputime_clock = CPUTIME_CLOCK_I8254; } - gp->profrate = i8254_freq << CPUTIME_CLOCK_I8254_SHIFT; -#if defined(I586_CPU) || defined(I686_CPU) if (cputime_clock == CPUTIME_CLOCK_TSC) { - gp->profrate = tsc_freq >> 1; + gp->profrate = freq >> 1; cputime_prof_active = 1; - } + } else + gp->profrate = i8254_freq << CPUTIME_CLOCK_I8254_SHIFT; #if defined(PERFMON) && defined(I586_PMC_GUPROF) - else if (cputime_clock == CPUTIME_CLOCK_I586_PMC) { + if (cputime_clock == CPUTIME_CLOCK_I586_PMC) { if (perfmon_avail() && perfmon_setup(0, cputime_clock_pmc_conf) == 0) { if (perfmon_start(0) != 0) @@ -325,6 +327,10 @@ startguprof(gp) } } #endif /* PERFMON && I586_PMC_GUPROF */ +#else /* !(I586_CPU || I686_CPU) */ + if (cputime_clock == CPUTIME_CLOCK_UNINITIALIZED) + cputime_clock = CPUTIME_CLOCK_I8254; + gp->profrate = i8254_freq << CPUTIME_CLOCK_I8254_SHIFT; #endif /* I586_CPU || I686_CPU */ cputime_bias = 0; cputime(); Modified: head/sys/pc98/pc98/machdep.c ============================================================================== --- head/sys/pc98/pc98/machdep.c Thu Apr 7 23:14:33 2011 (r220432) +++ head/sys/pc98/pc98/machdep.c Thu Apr 7 23:28:28 2011 (r220433) @@ -1072,16 +1072,17 @@ int cpu_est_clockrate(int cpu_id, uint64_t *rate) { register_t reg; - uint64_t tsc1, tsc2; + uint64_t freq, tsc1, tsc2; if (pcpu_find(cpu_id) == NULL || rate == NULL) return (EINVAL); if ((cpu_feature & CPUID_TSC) == 0) return (EOPNOTSUPP); + freq = atomic_load_acq_64(&tsc_freq); /* If we're booting, trust the rate calibrated moments ago. */ - if (cold && tsc_freq != 0) { - *rate = tsc_freq; + if (cold && freq != 0) { + *rate = freq; return (0); } @@ -1110,7 +1111,7 @@ cpu_est_clockrate(int cpu_id, uint64_t * #endif tsc2 -= tsc1; - if (tsc_freq != 0) { + if (freq != 0) { *rate = tsc2 * 1000; return (0); } Modified: head/sys/x86/cpufreq/est.c ============================================================================== --- head/sys/x86/cpufreq/est.c Thu Apr 7 23:14:33 2011 (r220432) +++ head/sys/x86/cpufreq/est.c Thu Apr 7 23:28:28 2011 (r220433) @@ -1215,7 +1215,7 @@ est_msr_info(device_t dev, uint64_t msr, return (EOPNOTSUPP); /* Figure out the bus clock. */ - freq = tsc_freq / 1000000; + freq = atomic_load_acq_64(&tsc_freq) / 1000000; id = msr >> 32; bus = freq / (id >> 8); device_printf(dev, "Guessed bus clock (high) of %d MHz\n", bus); Modified: head/sys/x86/isa/clock.c ============================================================================== --- head/sys/x86/isa/clock.c Thu Apr 7 23:14:33 2011 (r220432) +++ head/sys/x86/isa/clock.c Thu Apr 7 23:28:28 2011 (r220433) @@ -246,13 +246,13 @@ getit(void) } static __inline void -delay_tsc(int n) +delay_tsc(int n, uint64_t freq) { uint64_t start, end, now; sched_pin(); start = rdtsc(); - end = start + (tsc_freq * n) / 1000000; + end = start + (freq * n) / 1000000; do { cpu_spinwait(); now = rdtsc(); @@ -290,6 +290,7 @@ void DELAY(int n) { struct timecounter *tc; + uint64_t freq; int delta, prev_tick, tick, ticks_left; #ifdef DELAYDEBUG @@ -298,8 +299,9 @@ DELAY(int n) static int state = 0; #endif - if (tsc_freq != 0) { - delay_tsc(n); + freq = atomic_load_acq_64(&tsc_freq); + if (freq != 0) { + delay_tsc(n, freq); return; } tc = timecounter; Modified: head/sys/x86/x86/tsc.c ============================================================================== --- head/sys/x86/x86/tsc.c Thu Apr 7 23:14:33 2011 (r220432) +++ head/sys/x86/x86/tsc.c Thu Apr 7 23:28:28 2011 (r220433) @@ -245,14 +245,16 @@ tsc_freq_changing(void *arg, const struc static void tsc_freq_changed(void *arg, const struct cf_level *level, int status) { + uint64_t freq; /* If there was an error during the transition, don't do anything. */ if (tsc_disabled || status != 0) return; /* Total setting for this level gives the new frequency in MHz. */ - tsc_freq = (uint64_t)level->total_set.freq * 1000000; - tsc_timecounter.tc_frequency = tsc_freq; + freq = (uint64_t)level->total_set.freq * 1000000; + atomic_store_rel_64(&tsc_freq, freq); + atomic_store_rel_64(&tsc_timecounter.tc_frequency, freq); } static int @@ -261,13 +263,13 @@ sysctl_machdep_tsc_freq(SYSCTL_HANDLER_A int error; uint64_t freq; - if (tsc_timecounter.tc_frequency == 0) + freq = atomic_load_acq_64(&tsc_freq); + if (freq == 0) return (EOPNOTSUPP); - freq = tsc_freq; error = sysctl_handle_64(oidp, &freq, 0, req); if (error == 0 && req->newptr != NULL) { - tsc_freq = freq; - tsc_timecounter.tc_frequency = tsc_freq; + atomic_store_rel_64(&tsc_freq, freq); + atomic_store_rel_64(&tsc_timecounter.tc_frequency, freq); } return (error); }
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201104072328.p37NSSsO055101>