From owner-svn-src-head@FreeBSD.ORG Thu Jun 17 11:54:50 2010 Return-Path: Delivered-To: svn-src-head@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:4f8:fff6::34]) by hub.freebsd.org (Postfix) with ESMTP id 366E91065672; Thu, 17 Jun 2010 11:54:50 +0000 (UTC) (envelope-from mav@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:4f8:fff6::2c]) by mx1.freebsd.org (Postfix) with ESMTP id 23B9B8FC08; Thu, 17 Jun 2010 11:54:50 +0000 (UTC) Received: from svn.freebsd.org (localhost [127.0.0.1]) by svn.freebsd.org (8.14.3/8.14.3) with ESMTP id o5HBso0Q014591; Thu, 17 Jun 2010 11:54:50 GMT (envelope-from mav@svn.freebsd.org) Received: (from mav@localhost) by svn.freebsd.org (8.14.3/8.14.3/Submit) id o5HBsnTh014584; Thu, 17 Jun 2010 11:54:49 GMT (envelope-from mav@svn.freebsd.org) Message-Id: <201006171154.o5HBsnTh014584@svn.freebsd.org> From: Alexander Motin Date: Thu, 17 Jun 2010 11:54:49 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org X-SVN-Group: head MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cc: Subject: svn commit: r209248 - in head/sys: amd64/amd64 amd64/conf amd64/include conf i386/i386 X-BeenThere: svn-src-head@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: SVN commit messages for the src tree for head/-current List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 17 Jun 2010 11:54:50 -0000 Author: mav Date: Thu Jun 17 11:54:49 2010 New Revision: 209248 URL: http://svn.freebsd.org/changeset/base/209248 Log: Merge COUNT_XINVLTLB_HITS and COUNT_IPIS kernel options from i386 to amd64. This information can be very valuable for CPU sleep-time (and respectively idle power consumption) optimization. Add counters for timer-related IPIs. Reviewed by: jhb@ (previous version) Modified: head/sys/amd64/amd64/apic_vector.S head/sys/amd64/amd64/mp_machdep.c head/sys/amd64/conf/NOTES head/sys/amd64/include/smp.h head/sys/conf/options.amd64 head/sys/i386/i386/mp_machdep.c Modified: head/sys/amd64/amd64/apic_vector.S ============================================================================== --- head/sys/amd64/amd64/apic_vector.S Thu Jun 17 10:15:13 2010 (r209247) +++ head/sys/amd64/amd64/apic_vector.S Thu Jun 17 11:54:49 2010 (r209248) @@ -36,6 +36,8 @@ * as well as IPI handlers. */ +#include "opt_smp.h" + #include #include @@ -135,6 +137,19 @@ IDTVEC(errorint) .text SUPERALIGN_TEXT IDTVEC(invltlb) +#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS) + PUSH_FRAME + movl PCPU(CPUID), %eax +#ifdef COUNT_XINVLTLB_HITS + incl xhits_gbl(,%rax,4) +#endif +#ifdef COUNT_IPIS + movq ipi_invltlb_counts(,%rax,8),%rax + incq (%rax) +#endif + POP_FRAME +#endif + pushq %rax movq %cr3, %rax /* invalidate the TLB */ @@ -155,6 +170,19 @@ IDTVEC(invltlb) .text SUPERALIGN_TEXT IDTVEC(invlpg) +#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS) + PUSH_FRAME + movl PCPU(CPUID), %eax +#ifdef COUNT_XINVLTLB_HITS + incl xhits_pg(,%rax,4) +#endif +#ifdef COUNT_IPIS + movq ipi_invlpg_counts(,%rax,8),%rax + incq (%rax) +#endif + POP_FRAME +#endif + pushq %rax movq smp_tlb_addr1, %rax @@ -175,6 +203,19 @@ IDTVEC(invlpg) .text SUPERALIGN_TEXT IDTVEC(invlrng) +#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS) + PUSH_FRAME + movl PCPU(CPUID), %eax +#ifdef COUNT_XINVLTLB_HITS + incl xhits_rng(,%rax,4) +#endif +#ifdef COUNT_IPIS + movq ipi_invlrng_counts(,%rax,8),%rax + incq (%rax) +#endif + POP_FRAME +#endif + pushq %rax pushq %rdx @@ -201,6 +242,14 @@ IDTVEC(invlrng) .text SUPERALIGN_TEXT IDTVEC(invlcache) +#ifdef COUNT_IPIS + PUSH_FRAME + movl PCPU(CPUID), %eax + movq ipi_invlcache_counts(,%rax,8),%rax + incq (%rax) + POP_FRAME +#endif + pushq %rax wbinvd @@ -270,6 +319,11 @@ IDTVEC(cpususpend) SUPERALIGN_TEXT IDTVEC(rendezvous) PUSH_FRAME +#ifdef COUNT_IPIS + movl PCPU(CPUID), %eax + movq ipi_rendezvous_counts(,%rax,8), %rax + incq (%rax) +#endif call smp_rendezvous_action movq lapic, %rax movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */ Modified: head/sys/amd64/amd64/mp_machdep.c ============================================================================== --- head/sys/amd64/amd64/mp_machdep.c Thu Jun 17 10:15:13 2010 (r209247) +++ head/sys/amd64/amd64/mp_machdep.c Thu Jun 17 11:54:49 2010 (r209248) @@ -31,6 +31,7 @@ __FBSDID("$FreeBSD$"); #include "opt_kstack_pages.h" #include "opt_mp_watchdog.h" #include "opt_sched.h" +#include "opt_smp.h" #include #include @@ -106,6 +107,20 @@ vm_offset_t smp_tlb_addr1; vm_offset_t smp_tlb_addr2; volatile int smp_tlb_wait; +#ifdef COUNT_IPIS +/* Interrupt counts. */ +static u_long *ipi_preempt_counts[MAXCPU]; +static u_long *ipi_ast_counts[MAXCPU]; +u_long *ipi_invltlb_counts[MAXCPU]; +u_long *ipi_invlrng_counts[MAXCPU]; +u_long *ipi_invlpg_counts[MAXCPU]; +u_long *ipi_invlcache_counts[MAXCPU]; +u_long *ipi_rendezvous_counts[MAXCPU]; +u_long *ipi_lazypmap_counts[MAXCPU]; +static u_long *ipi_hardclock_counts[MAXCPU]; +static u_long *ipi_statclock_counts[MAXCPU]; +#endif + extern inthand_t IDTVEC(fast_syscall), IDTVEC(fast_syscall32); /* @@ -970,6 +985,42 @@ start_ap(int apic_id) return 0; /* return FAILURE */ } +#ifdef COUNT_XINVLTLB_HITS +u_int xhits_gbl[MAXCPU]; +u_int xhits_pg[MAXCPU]; +u_int xhits_rng[MAXCPU]; +SYSCTL_NODE(_debug, OID_AUTO, xhits, CTLFLAG_RW, 0, ""); +SYSCTL_OPAQUE(_debug_xhits, OID_AUTO, global, CTLFLAG_RW, &xhits_gbl, + sizeof(xhits_gbl), "IU", ""); +SYSCTL_OPAQUE(_debug_xhits, OID_AUTO, page, CTLFLAG_RW, &xhits_pg, + sizeof(xhits_pg), "IU", ""); +SYSCTL_OPAQUE(_debug_xhits, OID_AUTO, range, CTLFLAG_RW, &xhits_rng, + sizeof(xhits_rng), "IU", ""); + +u_int ipi_global; +u_int ipi_page; +u_int ipi_range; +u_int ipi_range_size; +SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_global, CTLFLAG_RW, &ipi_global, 0, ""); +SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_page, CTLFLAG_RW, &ipi_page, 0, ""); +SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_range, CTLFLAG_RW, &ipi_range, 0, ""); +SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_range_size, CTLFLAG_RW, &ipi_range_size, + 0, ""); + +u_int ipi_masked_global; +u_int ipi_masked_page; +u_int ipi_masked_range; +u_int ipi_masked_range_size; +SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_masked_global, CTLFLAG_RW, + &ipi_masked_global, 0, ""); +SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_masked_page, CTLFLAG_RW, + &ipi_masked_page, 0, ""); +SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_masked_range, CTLFLAG_RW, + &ipi_masked_range, 0, ""); +SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_masked_range_size, CTLFLAG_RW, + &ipi_masked_range_size, 0, ""); +#endif /* COUNT_XINVLTLB_HITS */ + /* * Flush the TLB on all other CPU's */ @@ -1047,6 +1098,9 @@ smp_invltlb(void) if (smp_started) { smp_tlb_shootdown(IPI_INVLTLB, 0, 0); +#ifdef COUNT_XINVLTLB_HITS + ipi_global++; +#endif } } @@ -1054,8 +1108,12 @@ void smp_invlpg(vm_offset_t addr) { - if (smp_started) + if (smp_started) { smp_tlb_shootdown(IPI_INVLPG, addr, 0); +#ifdef COUNT_XINVLTLB_HITS + ipi_page++; +#endif + } } void @@ -1064,6 +1122,10 @@ smp_invlpg_range(vm_offset_t addr1, vm_o if (smp_started) { smp_tlb_shootdown(IPI_INVLRNG, addr1, addr2); +#ifdef COUNT_XINVLTLB_HITS + ipi_range++; + ipi_range_size += (addr2 - addr1) / PAGE_SIZE; +#endif } } @@ -1073,6 +1135,9 @@ smp_masked_invltlb(cpumask_t mask) if (smp_started) { smp_targeted_tlb_shootdown(mask, IPI_INVLTLB, 0, 0); +#ifdef COUNT_XINVLTLB_HITS + ipi_masked_global++; +#endif } } @@ -1082,6 +1147,9 @@ smp_masked_invlpg(cpumask_t mask, vm_off if (smp_started) { smp_targeted_tlb_shootdown(mask, IPI_INVLPG, addr, 0); +#ifdef COUNT_XINVLTLB_HITS + ipi_masked_page++; +#endif } } @@ -1091,6 +1159,10 @@ smp_masked_invlpg_range(cpumask_t mask, if (smp_started) { smp_targeted_tlb_shootdown(mask, IPI_INVLRNG, addr1, addr2); +#ifdef COUNT_XINVLTLB_HITS + ipi_masked_range++; + ipi_masked_range_size += (addr2 - addr1) / PAGE_SIZE; +#endif } } @@ -1102,16 +1174,30 @@ ipi_bitmap_handler(struct trapframe fram ipi_bitmap = atomic_readandclear_int(&cpu_ipi_pending[cpu]); - if (ipi_bitmap & (1 << IPI_PREEMPT)) + if (ipi_bitmap & (1 << IPI_PREEMPT)) { +#ifdef COUNT_IPIS + (*ipi_preempt_counts[cpu])++; +#endif sched_preempt(curthread); - - /* Nothing to do for AST */ - - if (ipi_bitmap & (1 << IPI_HARDCLOCK)) + } + if (ipi_bitmap & (1 << IPI_AST)) { +#ifdef COUNT_IPIS + (*ipi_ast_counts[cpu])++; +#endif + /* Nothing to do for AST */ + } + if (ipi_bitmap & (1 << IPI_HARDCLOCK)) { +#ifdef COUNT_IPIS + (*ipi_hardclock_counts[cpu])++; +#endif hardclockintr(&frame); - - if (ipi_bitmap & (1 << IPI_STATCLOCK)) + } + if (ipi_bitmap & (1 << IPI_STATCLOCK)) { +#ifdef COUNT_IPIS + (*ipi_statclock_counts[cpu])++; +#endif statclockintr(&frame); + } } /* @@ -1432,3 +1518,38 @@ mp_grab_cpu_hlt(void) __asm __volatile("sti; hlt" : : : "memory"); return (retval); } + +#ifdef COUNT_IPIS +/* + * Setup interrupt counters for IPI handlers. + */ +static void +mp_ipi_intrcnt(void *dummy) +{ + char buf[64]; + int i; + + CPU_FOREACH(i) { + snprintf(buf, sizeof(buf), "cpu%d:invltlb", i); + intrcnt_add(buf, &ipi_invltlb_counts[i]); + snprintf(buf, sizeof(buf), "cpu%d:invlrng", i); + intrcnt_add(buf, &ipi_invlrng_counts[i]); + snprintf(buf, sizeof(buf), "cpu%d:invlpg", i); + intrcnt_add(buf, &ipi_invlpg_counts[i]); + snprintf(buf, sizeof(buf), "cpu%d:preempt", i); + intrcnt_add(buf, &ipi_preempt_counts[i]); + snprintf(buf, sizeof(buf), "cpu%d:ast", i); + intrcnt_add(buf, &ipi_ast_counts[i]); + snprintf(buf, sizeof(buf), "cpu%d:rendezvous", i); + intrcnt_add(buf, &ipi_rendezvous_counts[i]); + snprintf(buf, sizeof(buf), "cpu%d:lazypmap", i); + intrcnt_add(buf, &ipi_lazypmap_counts[i]); + snprintf(buf, sizeof(buf), "cpu%d:hardclock", i); + intrcnt_add(buf, &ipi_hardclock_counts[i]); + snprintf(buf, sizeof(buf), "cpu%d:statclock", i); + intrcnt_add(buf, &ipi_statclock_counts[i]); + } +} +SYSINIT(mp_ipi_intrcnt, SI_SUB_INTR, SI_ORDER_MIDDLE, mp_ipi_intrcnt, NULL); +#endif + Modified: head/sys/amd64/conf/NOTES ============================================================================== --- head/sys/amd64/conf/NOTES Thu Jun 17 10:15:13 2010 (r209247) +++ head/sys/amd64/conf/NOTES Thu Jun 17 11:54:49 2010 (r209248) @@ -30,6 +30,11 @@ device mptable # Optional MPSPEC mpta # options MP_WATCHDOG +# Debugging options. +# +options COUNT_XINVLTLB_HITS # Counters for TLB events +options COUNT_IPIS # Per-CPU IPI interrupt counters + ##################################################################### Modified: head/sys/amd64/include/smp.h ============================================================================== --- head/sys/amd64/include/smp.h Thu Jun 17 10:15:13 2010 (r209247) +++ head/sys/amd64/include/smp.h Thu Jun 17 11:54:49 2010 (r209248) @@ -35,6 +35,14 @@ extern int mp_naps; extern int boot_cpu_id; extern struct pcb stoppcbs[]; extern int cpu_apic_ids[]; +#ifdef COUNT_IPIS +extern u_long *ipi_invltlb_counts[MAXCPU]; +extern u_long *ipi_invlrng_counts[MAXCPU]; +extern u_long *ipi_invlpg_counts[MAXCPU]; +extern u_long *ipi_invlcache_counts[MAXCPU]; +extern u_long *ipi_rendezvous_counts[MAXCPU]; +extern u_long *ipi_lazypmap_counts[MAXCPU]; +#endif /* IPI handlers */ inthand_t Modified: head/sys/conf/options.amd64 ============================================================================== --- head/sys/conf/options.amd64 Thu Jun 17 10:15:13 2010 (r209247) +++ head/sys/conf/options.amd64 Thu Jun 17 11:54:49 2010 (r209248) @@ -3,6 +3,8 @@ AUTO_EOI_1 opt_auto_eoi.h AUTO_EOI_2 opt_auto_eoi.h +COUNT_XINVLTLB_HITS opt_smp.h +COUNT_IPIS opt_smp.h MAXMEM PERFMON PMAP_SHPGPERPROC opt_pmap.h Modified: head/sys/i386/i386/mp_machdep.c ============================================================================== --- head/sys/i386/i386/mp_machdep.c Thu Jun 17 10:15:13 2010 (r209247) +++ head/sys/i386/i386/mp_machdep.c Thu Jun 17 11:54:49 2010 (r209248) @@ -166,6 +166,8 @@ u_long *ipi_invlpg_counts[MAXCPU]; u_long *ipi_invlcache_counts[MAXCPU]; u_long *ipi_rendezvous_counts[MAXCPU]; u_long *ipi_lazypmap_counts[MAXCPU]; +static u_long *ipi_hardclock_counts[MAXCPU]; +static u_long *ipi_statclock_counts[MAXCPU]; #endif /* @@ -1266,19 +1268,24 @@ ipi_bitmap_handler(struct trapframe fram #endif sched_preempt(curthread); } - if (ipi_bitmap & (1 << IPI_AST)) { #ifdef COUNT_IPIS (*ipi_ast_counts[cpu])++; #endif /* Nothing to do for AST */ } - - if (ipi_bitmap & (1 << IPI_HARDCLOCK)) + if (ipi_bitmap & (1 << IPI_HARDCLOCK)) { +#ifdef COUNT_IPIS + (*ipi_hardclock_counts[cpu])++; +#endif hardclockintr(&frame); - - if (ipi_bitmap & (1 << IPI_STATCLOCK)) + } + if (ipi_bitmap & (1 << IPI_STATCLOCK)) { +#ifdef COUNT_IPIS + (*ipi_statclock_counts[cpu])++; +#endif statclockintr(&frame); + } } /* @@ -1574,20 +1581,24 @@ mp_ipi_intrcnt(void *dummy) int i; CPU_FOREACH(i) { - snprintf(buf, sizeof(buf), "cpu%d: invltlb", i); + snprintf(buf, sizeof(buf), "cpu%d:invltlb", i); intrcnt_add(buf, &ipi_invltlb_counts[i]); - snprintf(buf, sizeof(buf), "cpu%d: invlrng", i); + snprintf(buf, sizeof(buf), "cpu%d:invlrng", i); intrcnt_add(buf, &ipi_invlrng_counts[i]); - snprintf(buf, sizeof(buf), "cpu%d: invlpg", i); + snprintf(buf, sizeof(buf), "cpu%d:invlpg", i); intrcnt_add(buf, &ipi_invlpg_counts[i]); - snprintf(buf, sizeof(buf), "cpu%d: preempt", i); + snprintf(buf, sizeof(buf), "cpu%d:preempt", i); intrcnt_add(buf, &ipi_preempt_counts[i]); - snprintf(buf, sizeof(buf), "cpu%d: ast", i); + snprintf(buf, sizeof(buf), "cpu%d:ast", i); intrcnt_add(buf, &ipi_ast_counts[i]); - snprintf(buf, sizeof(buf), "cpu%d: rendezvous", i); + snprintf(buf, sizeof(buf), "cpu%d:rendezvous", i); intrcnt_add(buf, &ipi_rendezvous_counts[i]); - snprintf(buf, sizeof(buf), "cpu%d: lazypmap", i); + snprintf(buf, sizeof(buf), "cpu%d:lazypmap", i); intrcnt_add(buf, &ipi_lazypmap_counts[i]); + snprintf(buf, sizeof(buf), "cpu%d:hardclock", i); + intrcnt_add(buf, &ipi_hardclock_counts[i]); + snprintf(buf, sizeof(buf), "cpu%d:statclock", i); + intrcnt_add(buf, &ipi_statclock_counts[i]); } } SYSINIT(mp_ipi_intrcnt, SI_SUB_INTR, SI_ORDER_MIDDLE, mp_ipi_intrcnt, NULL);