Date: Mon, 28 Mar 2016 09:52:44 +0000 (UTC) From: Konstantin Belousov <kib@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r297348 - head/sys/x86/x86 Message-ID: <201603280952.u2S9qicL011534@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: kib Date: Mon Mar 28 09:52:44 2016 New Revision: 297348 URL: https://svnweb.freebsd.org/changeset/base/297348 Log: Use TSC deadline mode for LAPIC timer, when available. The mode fires LAPIC timer iinterrupt when TSC reaches the value written to the IA32_TSC_DEADLINE MSR. To arm or reset the timer in deadline mode, a single non-serializing MSR write is enough. This is an advance from the one-shot mode of LAPIC, where timer operated with the FSB frequency and required two (serialized in case of xAPIC) writes to the APIC registers. The LVT_TIMER register value is cached to avoid unneeded writes in the deadline mode. Unused arguments to specify period (which is passed in struct lapic as la_timer_period) and interrupt enable (which is always enabled) are removed from lapic_timer_{oneshot,periodic,deadline} functions. Instead, special lapic_timer_oneshot_nointr() function for interrupt-less one-shot calibration is added. Reviewed by: mav (previous version) Tested by: pho Sponsored by: The FreeBSD Foundation Differential revision: https://reviews.freebsd.org/D5738 Modified: head/sys/x86/x86/local_apic.c Modified: head/sys/x86/x86/local_apic.c ============================================================================== --- head/sys/x86/x86/local_apic.c Mon Mar 28 09:43:40 2016 (r297347) +++ head/sys/x86/x86/local_apic.c Mon Mar 28 09:52:44 2016 (r297348) @@ -56,6 +56,7 @@ __FBSDID("$FreeBSD$"); #include <vm/pmap.h> #include <x86/apicreg.h> +#include <machine/clock.h> #include <machine/cpufunc.h> #include <machine/cputypes.h> #include <machine/frame.h> @@ -94,6 +95,13 @@ CTASSERT(IPI_STOP < APIC_SPURIOUS_INT); #define IRQ_DTRACE_RET (NUM_IO_INTS + 3) #define IRQ_EVTCHN (NUM_IO_INTS + 4) +enum lat_timer_mode { + LAT_MODE_UNDEF = 0, + LAT_MODE_PERIODIC = 1, + LAT_MODE_ONESHOT = 2, + LAT_MODE_DEADLINE = 3, +}; + /* * Support for local APICs. Local APICs manage interrupts on each * individual processor as opposed to I/O APICs which receive interrupts @@ -119,9 +127,10 @@ struct lapic { u_int la_cluster_id:2; u_int la_present:1; u_long *la_timer_count; - u_long la_timer_period; - u_int la_timer_mode; - uint32_t lvt_timer_cache; + uint64_t la_timer_period; + enum lat_timer_mode la_timer_mode; + uint32_t lvt_timer_base; + uint32_t lvt_timer_last; /* Include IDT_SYSCALL to make indexing easier. */ int la_ioint_irqs[APIC_NUM_IOINTS + 1]; } static lapics[MAX_APIC_ID + 1]; @@ -160,6 +169,7 @@ volatile char *lapic_map; vm_paddr_t lapic_paddr; int x2apic_mode; int lapic_eoi_suppression; +static int lapic_timer_tsc_deadline; static u_long lapic_timer_divisor; static struct eventtimer lapic_et; @@ -167,6 +177,8 @@ SYSCTL_NODE(_hw, OID_AUTO, apic, CTLFLAG SYSCTL_INT(_hw_apic, OID_AUTO, x2apic_mode, CTLFLAG_RD, &x2apic_mode, 0, ""); SYSCTL_INT(_hw_apic, OID_AUTO, eoi_suppression, CTLFLAG_RD, &lapic_eoi_suppression, 0, ""); +SYSCTL_INT(_hw_apic, OID_AUTO, timer_tsc_deadline, CTLFLAG_RD, + &lapic_timer_tsc_deadline, 0, ""); static uint32_t lapic_read32(enum LAPIC_REGISTERS reg) @@ -256,10 +268,10 @@ native_lapic_enable_x2apic(void) static void lapic_enable(void); static void lapic_resume(struct pic *pic, bool suspend_cancelled); -static void lapic_timer_oneshot(struct lapic *, - u_int count, int enable_int); -static void lapic_timer_periodic(struct lapic *, - u_int count, int enable_int); +static void lapic_timer_oneshot(struct lapic *); +static void lapic_timer_oneshot_nointr(struct lapic *, uint32_t); +static void lapic_timer_periodic(struct lapic *); +static void lapic_timer_deadline(struct lapic *); static void lapic_timer_stop(struct lapic *); static void lapic_timer_set_divisor(u_int divisor); static uint32_t lvt_mode(struct lapic *la, u_int pin, uint32_t value); @@ -450,7 +462,14 @@ native_lapic_init(vm_paddr_t addr) if (!arat) { lapic_et.et_flags |= ET_FLAGS_C3STOP; lapic_et.et_quality -= 200; + } else if ((cpu_feature & CPUID_TSC) != 0 && + (cpu_feature2 & CPUID2_TSCDLT) != 0 && + tsc_is_invariant && tsc_freq != 0) { + lapic_timer_tsc_deadline = 1; + TUNABLE_INT_FETCH("hw.lapic_tsc_deadline", + &lapic_timer_tsc_deadline); } + lapic_et.et_frequency = 0; /* We don't know frequency yet, so trying to guess. */ lapic_et.et_min_period = 0x00001000LL; @@ -604,23 +623,34 @@ native_lapic_setup(int boot) } /* Program timer LVT and setup handler. */ - la->lvt_timer_cache = lvt_mode(la, APIC_LVT_TIMER, + la->lvt_timer_base = lvt_mode(la, APIC_LVT_TIMER, lapic_read32(LAPIC_LVT_TIMER)); - lapic_write32(LAPIC_LVT_TIMER, la->lvt_timer_cache); + la->lvt_timer_last = la->lvt_timer_base; + lapic_write32(LAPIC_LVT_TIMER, la->lvt_timer_base); if (boot) { snprintf(buf, sizeof(buf), "cpu%d:timer", PCPU_GET(cpuid)); intrcnt_add(buf, &la->la_timer_count); } /* Setup the timer if configured. */ - if (la->la_timer_mode != 0) { + if (la->la_timer_mode != LAT_MODE_UNDEF) { KASSERT(la->la_timer_period != 0, ("lapic%u: zero divisor", lapic_id())); lapic_timer_set_divisor(lapic_timer_divisor); - if (la->la_timer_mode == 1) - lapic_timer_periodic(la, la->la_timer_period, 1); - else - lapic_timer_oneshot(la, la->la_timer_period, 1); + switch (la->la_timer_mode) { + case LAT_MODE_PERIODIC: + lapic_timer_periodic(la); + break; + case LAT_MODE_ONESHOT: + lapic_timer_oneshot(la); + break; + case LAT_MODE_DEADLINE: + lapic_timer_deadline(la); + break; + default: + panic("corrupted la_timer_mode %p %d", la, + la->la_timer_mode); + } } /* Program error LVT and clear any existing errors. */ @@ -722,46 +752,73 @@ native_lapic_disable_pmc(void) #endif } +static void +lapic_calibrate_initcount(struct eventtimer *et, struct lapic *la) +{ + u_long value; + + /* Start off with a divisor of 2 (power on reset default). */ + lapic_timer_divisor = 2; + /* Try to calibrate the local APIC timer. */ + do { + lapic_timer_set_divisor(lapic_timer_divisor); + lapic_timer_oneshot_nointr(la, APIC_TIMER_MAX_COUNT); + DELAY(1000000); + value = APIC_TIMER_MAX_COUNT - lapic_read32(LAPIC_CCR_TIMER); + if (value != APIC_TIMER_MAX_COUNT) + break; + lapic_timer_divisor <<= 1; + } while (lapic_timer_divisor <= 128); + if (lapic_timer_divisor > 128) + panic("lapic: Divisor too big"); + if (bootverbose) { + printf("lapic: Divisor %lu, Frequency %lu Hz\n", + lapic_timer_divisor, value); + } + et->et_frequency = value; +} + +static void +lapic_calibrate_deadline(struct eventtimer *et, struct lapic *la __unused) +{ + + et->et_frequency = tsc_freq; + if (bootverbose) { + printf("lapic: deadline tsc mode, Frequency %ju Hz\n", + (uintmax_t)et->et_frequency); + } +} + static int lapic_et_start(struct eventtimer *et, sbintime_t first, sbintime_t period) { struct lapic *la; - u_long value; la = &lapics[PCPU_GET(apic_id)]; if (et->et_frequency == 0) { - /* Start off with a divisor of 2 (power on reset default). */ - lapic_timer_divisor = 2; - /* Try to calibrate the local APIC timer. */ - do { - lapic_timer_set_divisor(lapic_timer_divisor); - lapic_timer_oneshot(la, APIC_TIMER_MAX_COUNT, 0); - DELAY(1000000); - value = APIC_TIMER_MAX_COUNT - - lapic_read32(LAPIC_CCR_TIMER); - if (value != APIC_TIMER_MAX_COUNT) - break; - lapic_timer_divisor <<= 1; - } while (lapic_timer_divisor <= 128); - if (lapic_timer_divisor > 128) - panic("lapic: Divisor too big"); - if (bootverbose) - printf("lapic: Divisor %lu, Frequency %lu Hz\n", - lapic_timer_divisor, value); - et->et_frequency = value; + if (lapic_timer_tsc_deadline) + lapic_calibrate_deadline(et, la); + else + lapic_calibrate_initcount(et, la); et->et_min_period = (0x00000002LLU << 32) / et->et_frequency; et->et_max_period = (0xfffffffeLLU << 32) / et->et_frequency; } - if (la->la_timer_mode == 0) + if (la->la_timer_mode == LAT_MODE_UNDEF) lapic_timer_set_divisor(lapic_timer_divisor); if (period != 0) { - la->la_timer_mode = 1; - la->la_timer_period = ((uint32_t)et->et_frequency * period) >> 32; - lapic_timer_periodic(la, la->la_timer_period, 1); + la->la_timer_mode = LAT_MODE_PERIODIC; + la->la_timer_period = ((uint32_t)et->et_frequency * period) >> + 32; + lapic_timer_periodic(la); + } else if (lapic_timer_tsc_deadline) { + la->la_timer_mode = LAT_MODE_DEADLINE; + la->la_timer_period = (et->et_frequency * first) >> 32; + lapic_timer_deadline(la); } else { - la->la_timer_mode = 2; - la->la_timer_period = ((uint32_t)et->et_frequency * first) >> 32; - lapic_timer_oneshot(la, la->la_timer_period, 1); + la->la_timer_mode = LAT_MODE_ONESHOT; + la->la_timer_period = ((uint32_t)et->et_frequency * first) >> + 32; + lapic_timer_oneshot(la); } return (0); } @@ -769,10 +826,11 @@ lapic_et_start(struct eventtimer *et, sb static int lapic_et_stop(struct eventtimer *et) { - struct lapic *la = &lapics[PCPU_GET(apic_id)]; + struct lapic *la; - la->la_timer_mode = 0; + la = &lapics[PCPU_GET(apic_id)]; lapic_timer_stop(la); + la->la_timer_mode = LAT_MODE_UNDEF; return (0); } @@ -1071,42 +1129,76 @@ lapic_timer_set_divisor(u_int divisor) } static void -lapic_timer_oneshot(struct lapic *la, u_int count, int enable_int) +lapic_timer_oneshot(struct lapic *la) { uint32_t value; - value = la->lvt_timer_cache; - value &= ~APIC_LVTT_TM; + value = la->lvt_timer_base; + value &= ~(APIC_LVTT_TM | APIC_LVT_M); value |= APIC_LVTT_TM_ONE_SHOT; - if (enable_int) - value &= ~APIC_LVT_M; + la->lvt_timer_last = value; lapic_write32(LAPIC_LVT_TIMER, value); - lapic_write32(LAPIC_ICR_TIMER, count); + lapic_write32(LAPIC_ICR_TIMER, la->la_timer_period); } static void -lapic_timer_periodic(struct lapic *la, u_int count, int enable_int) +lapic_timer_oneshot_nointr(struct lapic *la, uint32_t count) { uint32_t value; - value = la->lvt_timer_cache; + value = la->lvt_timer_base; value &= ~APIC_LVTT_TM; - value |= APIC_LVTT_TM_PERIODIC; - if (enable_int) - value &= ~APIC_LVT_M; + value |= APIC_LVTT_TM_ONE_SHOT | APIC_LVT_M; + la->lvt_timer_last = value; lapic_write32(LAPIC_LVT_TIMER, value); lapic_write32(LAPIC_ICR_TIMER, count); } static void -lapic_timer_stop(struct lapic *la) +lapic_timer_periodic(struct lapic *la) { uint32_t value; - value = la->lvt_timer_cache; - value &= ~APIC_LVTT_TM; - value |= APIC_LVT_M; + value = la->lvt_timer_base; + value &= ~(APIC_LVTT_TM | APIC_LVT_M); + value |= APIC_LVTT_TM_PERIODIC; + la->lvt_timer_last = value; lapic_write32(LAPIC_LVT_TIMER, value); + lapic_write32(LAPIC_ICR_TIMER, la->la_timer_period); +} + +static void +lapic_timer_deadline(struct lapic *la) +{ + uint32_t value; + + value = la->lvt_timer_base; + value &= ~(APIC_LVTT_TM | APIC_LVT_M); + value |= APIC_LVTT_TM_TSCDLT; + if (value != la->lvt_timer_last) { + la->lvt_timer_last = value; + lapic_write32_nofence(LAPIC_LVT_TIMER, value); + if (!x2apic_mode) + mfence(); + } + wrmsr(MSR_TSC_DEADLINE, la->la_timer_period + rdtsc()); +} + +static void +lapic_timer_stop(struct lapic *la) +{ + uint32_t value; + + if (la->la_timer_mode == LAT_MODE_DEADLINE) { + wrmsr(MSR_TSC_DEADLINE, 0); + mfence(); + } else { + value = la->lvt_timer_base; + value &= ~APIC_LVTT_TM; + value |= APIC_LVT_M; + la->lvt_timer_last = value; + lapic_write32(LAPIC_LVT_TIMER, value); + } } void
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201603280952.u2S9qicL011534>