From owner-svn-src-head@FreeBSD.ORG Tue Mar 11 10:03:32 2014 Return-Path: Delivered-To: svn-src-head@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [8.8.178.115]) (using TLSv1 with cipher ADH-AES256-SHA (256/256 bits)) (No client certificate requested) by hub.freebsd.org (Postfix) with ESMTPS id 080D4F7A; Tue, 11 Mar 2014 10:03:32 +0000 (UTC) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:1900:2254:2068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by mx1.freebsd.org (Postfix) with ESMTPS id E692BDFF; Tue, 11 Mar 2014 10:03:31 +0000 (UTC) Received: from svn.freebsd.org ([127.0.1.70]) by svn.freebsd.org (8.14.8/8.14.8) with ESMTP id s2BA3VG8014419; Tue, 11 Mar 2014 10:03:31 GMT (envelope-from royger@svn.freebsd.org) Received: (from royger@localhost) by svn.freebsd.org (8.14.8/8.14.8/Submit) id s2BA3U0c014411; Tue, 11 Mar 2014 10:03:30 GMT (envelope-from royger@svn.freebsd.org) Message-Id: <201403111003.s2BA3U0c014411@svn.freebsd.org> From: Roger Pau Monné Date: Tue, 11 Mar 2014 10:03:30 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r263001 - in head/sys: amd64/amd64 amd64/include i386/i386 i386/include i386/xen x86/xen X-SVN-Group: head MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-head@freebsd.org X-Mailman-Version: 2.1.17 Precedence: list List-Id: SVN commit messages for the src tree for head/-current List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Tue, 11 Mar 2014 10:03:32 -0000 Author: royger Date: Tue Mar 11 10:03:29 2014 New Revision: 263001 URL: http://svnweb.freebsd.org/changeset/base/263001 Log: amd64/i386: switch IPI handlers to C code. Move asm IPIs handlers to C code, so both Xen and native IPI handlers share the same code. Reviewed by: jhb Approved by: gibbs Sponsored by: Citrix Systems R&D amd64/amd64/apic_vector.S: i386/i386/apic_vector.s: - Remove asm coded IPI handlers and instead call the newly introduced C variants. amd64/amd64/mp_machdep.c: i386/i386/mp_machdep.c: - Add C coded clones to the asm IPI handlers (moved from x86/xen/hvm.c). i386/include/smp.h: amd64/include/smp.h: - Add prototypes for the C IPI handlers. x86/xen/hvm.c: - Move the C IPI handlers to mp_machdep and call those in the Xen IPI handlers. i386/xen/mp_machdep.c: - Add dummy IPI handlers to the i386 Xen PV port (this port doesn't support SMP). Modified: head/sys/amd64/amd64/apic_vector.S head/sys/amd64/amd64/mp_machdep.c head/sys/amd64/include/smp.h head/sys/i386/i386/apic_vector.s head/sys/i386/i386/mp_machdep.c head/sys/i386/include/smp.h head/sys/i386/xen/mp_machdep.c head/sys/x86/xen/hvm.c Modified: head/sys/amd64/amd64/apic_vector.S ============================================================================== --- head/sys/amd64/amd64/apic_vector.S Tue Mar 11 07:33:33 2014 (r263000) +++ head/sys/amd64/amd64/apic_vector.S Tue Mar 11 10:03:29 2014 (r263001) @@ -159,101 +159,26 @@ IDTVEC(xen_intr_upcall) #define NAKE_INTR_CS 24 SUPERALIGN_TEXT -global_invltlb: - movq %cr4,%rax - andq $~0x80,%rax /* PGE */ - movq %rax,%cr4 - orq $0x80,%rax - movq %rax,%cr4 -invltlb_ret_clear_pm_save: - movq smp_tlb_pmap,%rdx - testq %rdx,%rdx - jz invltlb_ret_rdx - testb $SEL_RPL_MASK,NAKE_INTR_CS(%rsp) - jz 1f - swapgs -1: - movl PCPU(CPUID),%eax - jz 2f - swapgs -2: - LK btcl %eax,PM_SAVE(%rdx) - SUPERALIGN_TEXT -invltlb_ret_rdx: - popq %rdx -invltlb_ret_rax: +invltlb_ret: movq lapic, %rax movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */ - LK incl smp_tlb_wait - popq %rax + POP_FRAME jmp doreti_iret SUPERALIGN_TEXT IDTVEC(invltlb_pcid) -#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS) PUSH_FRAME - movl PCPU(CPUID), %eax -#ifdef COUNT_XINVLTLB_HITS - incl xhits_gbl(,%rax,4) -#endif -#ifdef COUNT_IPIS - movq ipi_invltlb_counts(,%rax,8),%rax - incq (%rax) -#endif - POP_FRAME -#endif - - pushq %rax - pushq %rdx - movq %cr3,%rax + call invltlb_pcid_handler + jmp invltlb_ret - movq $smp_tlb_invpcid,%rdx - cmpl $0,(%rdx) - je global_invltlb - cmpl $-1,(%rdx) - je global_invltlb - - /* - * Only invalidate TLB for entries with current PCID. - */ - cmpl $0,invpcid_works - je 1f - /* Use invpcid if available. */ - movl $1,%eax /* INVPCID_CTX */ - /* invpcid (%rdx),%rax */ - .byte 0x66,0x0f,0x38,0x82,0x02 - jmp invltlb_ret_clear_pm_save -1: - /* Otherwise reload %cr3 twice. */ - movq pcid_cr3,%rdx - cmpq %rax,%rdx - je 2f - movq %rdx,%cr3 /* Invalidate, bit 63 is zero. */ - btsq $63,%rax -2: - movq %rax,%cr3 - jmp invltlb_ret_clear_pm_save SUPERALIGN_TEXT IDTVEC(invltlb) -#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS) PUSH_FRAME - movl PCPU(CPUID), %eax -#ifdef COUNT_XINVLTLB_HITS - incl xhits_gbl(,%rax,4) -#endif -#ifdef COUNT_IPIS - movq ipi_invltlb_counts(,%rax,8),%rax - incq (%rax) -#endif - POP_FRAME -#endif - pushq %rax - movq %cr3, %rax /* invalidate the TLB */ - movq %rax, %cr3 - jmp invltlb_ret_rax + call invltlb_handler + jmp invltlb_ret /* * Single page TLB shootdown @@ -261,86 +186,17 @@ IDTVEC(invltlb) .text SUPERALIGN_TEXT IDTVEC(invlpg_pcid) -#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS) PUSH_FRAME - movl PCPU(CPUID), %eax -#ifdef COUNT_XINVLTLB_HITS - incl xhits_pg(,%rax,4) -#endif -#ifdef COUNT_IPIS - movq ipi_invlpg_counts(,%rax,8),%rax - incq (%rax) -#endif - POP_FRAME -#endif - pushq %rax - pushq %rdx - movq $smp_tlb_invpcid,%rdx - cmpl $0,invpcid_works - jne 2f - - /* kernel pmap - use invlpg to invalidate global mapping */ - cmpl $0,(%rdx) - je 3f - cmpl $-1,(%rdx) - je global_invltlb - - /* - * PCID supported, but INVPCID is not. - * Temporarily switch to the target address space and do INVLPG. - */ - pushq %rcx - movq %cr3,%rcx - movq pcid_cr3,%rax - cmp %rcx,%rax - je 1f - btsq $63,%rax - movq %rax,%cr3 -1: movq 8(%rdx),%rax - invlpg (%rax) - btsq $63,%rcx - movq %rcx,%cr3 - popq %rcx - jmp invltlb_ret_rdx - - /* - * Invalidate the TLB entry using INVPCID_ADDR. - */ -2: - xorl %eax,%eax -/* invpcid (%rdx),%rax */ - .byte 0x66,0x0f,0x38,0x82,0x02 - jmp invltlb_ret_rdx - - /* - * PCID is not supported or kernel pmap. - * Invalidate single page using INVLPG. - */ -3: - movq 8(%rdx),%rax - invlpg (%rax) - jmp invltlb_ret_rdx + call invlpg_pcid_handler + jmp invltlb_ret SUPERALIGN_TEXT IDTVEC(invlpg) -#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS) PUSH_FRAME - movl PCPU(CPUID), %eax -#ifdef COUNT_XINVLTLB_HITS - incl xhits_pg(,%rax,4) -#endif -#ifdef COUNT_IPIS - movq ipi_invlpg_counts(,%rax,8),%rax - incq (%rax) -#endif - POP_FRAME -#endif - pushq %rax - movq smp_tlb_invpcid+8,%rax - invlpg (%rax) /* invalidate single page */ - jmp invltlb_ret_rax + call invlpg_handler + jmp invltlb_ret /* * Page range TLB shootdown. @@ -348,81 +204,10 @@ IDTVEC(invlpg) .text SUPERALIGN_TEXT IDTVEC(invlrng) -#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS) PUSH_FRAME - movl PCPU(CPUID), %eax -#ifdef COUNT_XINVLTLB_HITS - incl xhits_rng(,%rax,4) -#endif -#ifdef COUNT_IPIS - movq ipi_invlrng_counts(,%rax,8),%rax - incq (%rax) -#endif - POP_FRAME -#endif - pushq %rax - pushq %rdx - movq $smp_tlb_invpcid,%rdx - cmpl $0,pmap_pcid_enabled - je invlrng_single_page - - /* kernel pmap - use invlpg to invalidate global mapping */ - cmpl $0,(%rdx) - je invlrng_single_page - cmpl $-1,(%rdx) - je global_invltlb - cmpl $0,invpcid_works - jne invlrng_invpcid - - pushq %rcx - movq %cr3,%rcx - movq pcid_cr3,%rax - cmpq %rcx,%rax - je 1f - btsq $63,%rax - movq %rax,%cr3 -1: - movq 8(%rdx),%rdx - movq smp_tlb_addr2,%rax -2: - invlpg (%rdx) - addq $PAGE_SIZE,%rdx - cmpq %rax,%rdx - jb 2b - btsq $63,%rcx - movq %rcx,%cr3 - popq %rcx - jmp invltlb_ret_rdx - -invlrng_invpcid: - pushq %rcx - subq $16,%rsp - movq (%rdx),%rcx - movq %rcx,(%rsp) - movq 8(%rdx),%rax - movq %rax,8(%rsp) - movq smp_tlb_addr2,%rcx - subq %rax,%rcx - shrq $PAGE_SHIFT,%rcx -1: -// invpcid (%rdx),%rax - .byte 0x66,0x0f,0x38,0x82,0x02 - addq $PAGE_SIZE,8(%rsp) - dec %rcx - jne 1b - addq $16,%rsp - popq %rcx - jmp invltlb_ret_rdx - -invlrng_single_page: - movq 8(%rdx),%rdx - movq smp_tlb_addr2,%rax -1: invlpg (%rdx) /* invalidate single page */ - addq $PAGE_SIZE,%rdx - cmpq %rax,%rdx - jb 1b - jmp invltlb_ret_rdx + call invlrng_handler + jmp invltlb_ret /* * Invalidate cache. @@ -430,17 +215,10 @@ invlrng_single_page: .text SUPERALIGN_TEXT IDTVEC(invlcache) -#ifdef COUNT_IPIS PUSH_FRAME - movl PCPU(CPUID), %eax - movq ipi_invlcache_counts(,%rax,8),%rax - incq (%rax) - POP_FRAME -#endif - pushq %rax - wbinvd - jmp invltlb_ret_rax + call invlcache_handler + jmp invltlb_ret /* * Handler for IPIs sent via the per-cpu IPI bitmap. Modified: head/sys/amd64/amd64/mp_machdep.c ============================================================================== --- head/sys/amd64/amd64/mp_machdep.c Tue Mar 11 07:33:33 2014 (r263000) +++ head/sys/amd64/amd64/mp_machdep.c Tue Mar 11 10:03:29 2014 (r263001) @@ -109,6 +109,7 @@ struct invpcid_descr smp_tlb_invpcid; volatile int smp_tlb_wait; uint64_t pcid_cr3; pmap_t smp_tlb_pmap; +extern int invpcid_works; #ifdef COUNT_IPIS /* Interrupt counts. */ @@ -1496,6 +1497,175 @@ cpususpend_handler(void) } /* + * Handlers for TLB related IPIs + */ +void +invltlb_handler(void) +{ +#ifdef COUNT_XINVLTLB_HITS + xhits_gbl[PCPU_GET(cpuid)]++; +#endif /* COUNT_XINVLTLB_HITS */ +#ifdef COUNT_IPIS + (*ipi_invltlb_counts[PCPU_GET(cpuid)])++; +#endif /* COUNT_IPIS */ + + invltlb(); + atomic_add_int(&smp_tlb_wait, 1); +} + +void +invltlb_pcid_handler(void) +{ + uint64_t cr3; +#ifdef COUNT_XINVLTLB_HITS + xhits_gbl[PCPU_GET(cpuid)]++; +#endif /* COUNT_XINVLTLB_HITS */ +#ifdef COUNT_IPIS + (*ipi_invltlb_counts[PCPU_GET(cpuid)])++; +#endif /* COUNT_IPIS */ + + cr3 = rcr3(); + if (smp_tlb_invpcid.pcid != (uint64_t)-1 && + smp_tlb_invpcid.pcid != 0) { + + if (invpcid_works) { + invpcid(&smp_tlb_invpcid, INVPCID_CTX); + } else { + /* Otherwise reload %cr3 twice. */ + if (cr3 != pcid_cr3) { + load_cr3(pcid_cr3); + cr3 |= CR3_PCID_SAVE; + } + load_cr3(cr3); + } + } else { + invltlb_globpcid(); + } + if (smp_tlb_pmap != NULL) + CPU_CLR_ATOMIC(PCPU_GET(cpuid), &smp_tlb_pmap->pm_save); + + atomic_add_int(&smp_tlb_wait, 1); +} + +void +invlpg_handler(void) +{ +#ifdef COUNT_XINVLTLB_HITS + xhits_pg[PCPU_GET(cpuid)]++; +#endif /* COUNT_XINVLTLB_HITS */ +#ifdef COUNT_IPIS + (*ipi_invlpg_counts[PCPU_GET(cpuid)])++; +#endif /* COUNT_IPIS */ + + invlpg(smp_tlb_invpcid.addr); + atomic_add_int(&smp_tlb_wait, 1); +} + +void +invlpg_pcid_handler(void) +{ +#ifdef COUNT_XINVLTLB_HITS + xhits_pg[PCPU_GET(cpuid)]++; +#endif /* COUNT_XINVLTLB_HITS */ +#ifdef COUNT_IPIS + (*ipi_invlpg_counts[PCPU_GET(cpuid)])++; +#endif /* COUNT_IPIS */ + + if (invpcid_works) { + invpcid(&smp_tlb_invpcid, INVPCID_ADDR); + } else if (smp_tlb_invpcid.pcid == 0) { + invlpg(smp_tlb_invpcid.addr); + } else if (smp_tlb_invpcid.pcid == (uint64_t)-1) { + invltlb_globpcid(); + } else { + uint64_t cr3; + + /* + * PCID supported, but INVPCID is not. + * Temporarily switch to the target address + * space and do INVLPG. + */ + cr3 = rcr3(); + if (cr3 != pcid_cr3) + load_cr3(pcid_cr3 | CR3_PCID_SAVE); + invlpg(smp_tlb_invpcid.addr); + load_cr3(cr3 | CR3_PCID_SAVE); + } + + atomic_add_int(&smp_tlb_wait, 1); +} + +static inline void +invlpg_range(vm_offset_t start, vm_offset_t end) +{ + + do { + invlpg(start); + start += PAGE_SIZE; + } while (start < end); +} + +void +invlrng_handler(void) +{ + vm_offset_t addr; +#ifdef COUNT_XINVLTLB_HITS + xhits_rng[PCPU_GET(cpuid)]++; +#endif /* COUNT_XINVLTLB_HITS */ +#ifdef COUNT_IPIS + (*ipi_invlrng_counts[PCPU_GET(cpuid)])++; +#endif /* COUNT_IPIS */ + + addr = smp_tlb_invpcid.addr; + if (pmap_pcid_enabled) { + if (invpcid_works) { + struct invpcid_descr d; + + d = smp_tlb_invpcid; + do { + invpcid(&d, INVPCID_ADDR); + d.addr += PAGE_SIZE; + } while (d.addr < smp_tlb_addr2); + } else if (smp_tlb_invpcid.pcid == 0) { + /* + * kernel pmap - use invlpg to invalidate + * global mapping. + */ + invlpg_range(addr, smp_tlb_addr2); + } else if (smp_tlb_invpcid.pcid == (uint64_t)-1) { + invltlb_globpcid(); + if (smp_tlb_pmap != NULL) { + CPU_CLR_ATOMIC(PCPU_GET(cpuid), + &smp_tlb_pmap->pm_save); + } + } else { + uint64_t cr3; + + cr3 = rcr3(); + if (cr3 != pcid_cr3) + load_cr3(pcid_cr3 | CR3_PCID_SAVE); + invlpg_range(addr, smp_tlb_addr2); + load_cr3(cr3 | CR3_PCID_SAVE); + } + } else { + invlpg_range(addr, smp_tlb_addr2); + } + + atomic_add_int(&smp_tlb_wait, 1); +} + +void +invlcache_handler(void) +{ +#ifdef COUNT_IPIS + (*ipi_invlcache_counts[PCPU_GET(cpuid)])++; +#endif /* COUNT_IPIS */ + + wbinvd(); + atomic_add_int(&smp_tlb_wait, 1); +} + +/* * This is called once the rest of the system is up and running and we're * ready to let the AP's out of the pen. */ Modified: head/sys/amd64/include/smp.h ============================================================================== --- head/sys/amd64/include/smp.h Tue Mar 11 07:33:33 2014 (r263000) +++ head/sys/amd64/include/smp.h Tue Mar 11 10:03:29 2014 (r263001) @@ -62,6 +62,12 @@ struct pmap; void cpu_add(u_int apic_id, char boot_cpu); void cpustop_handler(void); void cpususpend_handler(void); +void invltlb_handler(void); +void invltlb_pcid_handler(void); +void invlpg_handler(void); +void invlpg_pcid_handler(void); +void invlrng_handler(void); +void invlcache_handler(void); void init_secondary(void); void ipi_startup(int apic_id, int vector); void ipi_all_but_self(u_int ipi); Modified: head/sys/i386/i386/apic_vector.s ============================================================================== --- head/sys/i386/i386/apic_vector.s Tue Mar 11 07:33:33 2014 (r263000) +++ head/sys/i386/i386/apic_vector.s Tue Mar 11 10:03:29 2014 (r263001) @@ -163,39 +163,21 @@ IDTVEC(xen_intr_upcall) */ .text SUPERALIGN_TEXT -IDTVEC(invltlb) - pushl %eax - pushl %ds - movl $KDSEL, %eax /* Kernel data selector */ - movl %eax, %ds - -#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS) - pushl %fs - movl $KPSEL, %eax /* Private space selector */ - movl %eax, %fs - movl PCPU(CPUID), %eax - popl %fs -#ifdef COUNT_XINVLTLB_HITS - incl xhits_gbl(,%eax,4) -#endif -#ifdef COUNT_IPIS - movl ipi_invltlb_counts(,%eax,4),%eax - incl (%eax) -#endif -#endif - - movl %cr3, %eax /* invalidate the TLB */ - movl %eax, %cr3 - +invltlb_ret: movl lapic, %eax movl $0, LA_EOI(%eax) /* End Of Interrupt to APIC */ + POP_FRAME + iret + + SUPERALIGN_TEXT +IDTVEC(invltlb) + PUSH_FRAME + SET_KERNEL_SREGS + cld - lock - incl smp_tlb_wait + call invltlb_handler - popl %ds - popl %eax - iret + jmp invltlb_ret /* * Single page TLB shootdown @@ -203,38 +185,13 @@ IDTVEC(invltlb) .text SUPERALIGN_TEXT IDTVEC(invlpg) - pushl %eax - pushl %ds - movl $KDSEL, %eax /* Kernel data selector */ - movl %eax, %ds - -#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS) - pushl %fs - movl $KPSEL, %eax /* Private space selector */ - movl %eax, %fs - movl PCPU(CPUID), %eax - popl %fs -#ifdef COUNT_XINVLTLB_HITS - incl xhits_pg(,%eax,4) -#endif -#ifdef COUNT_IPIS - movl ipi_invlpg_counts(,%eax,4),%eax - incl (%eax) -#endif -#endif - - movl smp_tlb_addr1, %eax - invlpg (%eax) /* invalidate single page */ - - movl lapic, %eax - movl $0, LA_EOI(%eax) /* End Of Interrupt to APIC */ + PUSH_FRAME + SET_KERNEL_SREGS + cld - lock - incl smp_tlb_wait + call invlpg_handler - popl %ds - popl %eax - iret + jmp invltlb_ret /* * Page range TLB shootdown. @@ -242,44 +199,13 @@ IDTVEC(invlpg) .text SUPERALIGN_TEXT IDTVEC(invlrng) - pushl %eax - pushl %edx - pushl %ds - movl $KDSEL, %eax /* Kernel data selector */ - movl %eax, %ds - -#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS) - pushl %fs - movl $KPSEL, %eax /* Private space selector */ - movl %eax, %fs - movl PCPU(CPUID), %eax - popl %fs -#ifdef COUNT_XINVLTLB_HITS - incl xhits_rng(,%eax,4) -#endif -#ifdef COUNT_IPIS - movl ipi_invlrng_counts(,%eax,4),%eax - incl (%eax) -#endif -#endif + PUSH_FRAME + SET_KERNEL_SREGS + cld - movl smp_tlb_addr1, %edx - movl smp_tlb_addr2, %eax -1: invlpg (%edx) /* invalidate single page */ - addl $PAGE_SIZE, %edx - cmpl %eax, %edx - jb 1b + call invlrng_handler - movl lapic, %eax - movl $0, LA_EOI(%eax) /* End Of Interrupt to APIC */ - - lock - incl smp_tlb_wait - - popl %ds - popl %edx - popl %eax - iret + jmp invltlb_ret /* * Invalidate cache. @@ -287,32 +213,13 @@ IDTVEC(invlrng) .text SUPERALIGN_TEXT IDTVEC(invlcache) - pushl %eax - pushl %ds - movl $KDSEL, %eax /* Kernel data selector */ - movl %eax, %ds - -#ifdef COUNT_IPIS - pushl %fs - movl $KPSEL, %eax /* Private space selector */ - movl %eax, %fs - movl PCPU(CPUID), %eax - popl %fs - movl ipi_invlcache_counts(,%eax,4),%eax - incl (%eax) -#endif - - wbinvd - - movl lapic, %eax - movl $0, LA_EOI(%eax) /* End Of Interrupt to APIC */ + PUSH_FRAME + SET_KERNEL_SREGS + cld - lock - incl smp_tlb_wait + call invlcache_handler - popl %ds - popl %eax - iret + jmp invltlb_ret /* * Handler for IPIs sent via the per-cpu IPI bitmap. Modified: head/sys/i386/i386/mp_machdep.c ============================================================================== --- head/sys/i386/i386/mp_machdep.c Tue Mar 11 07:33:33 2014 (r263000) +++ head/sys/i386/i386/mp_machdep.c Tue Mar 11 10:03:29 2014 (r263001) @@ -1551,6 +1551,72 @@ cpususpend_handler(void) CPU_CLR_ATOMIC(cpu, &suspended_cpus); CPU_CLR_ATOMIC(cpu, &started_cpus); } + +/* + * Handlers for TLB related IPIs + */ +void +invltlb_handler(void) +{ + uint64_t cr3; +#ifdef COUNT_XINVLTLB_HITS + xhits_gbl[PCPU_GET(cpuid)]++; +#endif /* COUNT_XINVLTLB_HITS */ +#ifdef COUNT_IPIS + (*ipi_invltlb_counts[PCPU_GET(cpuid)])++; +#endif /* COUNT_IPIS */ + + cr3 = rcr3(); + load_cr3(cr3); + atomic_add_int(&smp_tlb_wait, 1); +} + +void +invlpg_handler(void) +{ +#ifdef COUNT_XINVLTLB_HITS + xhits_pg[PCPU_GET(cpuid)]++; +#endif /* COUNT_XINVLTLB_HITS */ +#ifdef COUNT_IPIS + (*ipi_invlpg_counts[PCPU_GET(cpuid)])++; +#endif /* COUNT_IPIS */ + + invlpg(smp_tlb_addr1); + + atomic_add_int(&smp_tlb_wait, 1); +} + +void +invlrng_handler(void) +{ + vm_offset_t addr; +#ifdef COUNT_XINVLTLB_HITS + xhits_rng[PCPU_GET(cpuid)]++; +#endif /* COUNT_XINVLTLB_HITS */ +#ifdef COUNT_IPIS + (*ipi_invlrng_counts[PCPU_GET(cpuid)])++; +#endif /* COUNT_IPIS */ + + addr = smp_tlb_addr1; + do { + invlpg(addr); + addr += PAGE_SIZE; + } while (addr < smp_tlb_addr2); + + atomic_add_int(&smp_tlb_wait, 1); +} + +void +invlcache_handler(void) +{ +#ifdef COUNT_IPIS + (*ipi_invlcache_counts[PCPU_GET(cpuid)])++; +#endif /* COUNT_IPIS */ + + wbinvd(); + atomic_add_int(&smp_tlb_wait, 1); +} + /* * This is called once the rest of the system is up and running and we're * ready to let the AP's out of the pen. Modified: head/sys/i386/include/smp.h ============================================================================== --- head/sys/i386/include/smp.h Tue Mar 11 07:33:33 2014 (r263000) +++ head/sys/i386/include/smp.h Tue Mar 11 10:03:29 2014 (r263001) @@ -63,6 +63,10 @@ void cpustop_handler(void); #ifndef XEN void cpususpend_handler(void); #endif +void invltlb_handler(void); +void invlpg_handler(void); +void invlrng_handler(void); +void invlcache_handler(void); void init_secondary(void); void ipi_startup(int apic_id, int vector); void ipi_all_but_self(u_int ipi); Modified: head/sys/i386/xen/mp_machdep.c ============================================================================== --- head/sys/i386/xen/mp_machdep.c Tue Mar 11 07:33:33 2014 (r263000) +++ head/sys/i386/xen/mp_machdep.c Tue Mar 11 10:03:29 2014 (r263001) @@ -1276,6 +1276,31 @@ cpustop_handler(void) } /* + * Handlers for TLB related IPIs + * + * On i386 Xen PV this are no-ops since this port doesn't support SMP. + */ +void +invltlb_handler(void) +{ +} + +void +invlpg_handler(void) +{ +} + +void +invlrng_handler(void) +{ +} + +void +invlcache_handler(void) +{ +} + +/* * This is called once the rest of the system is up and running and we're * ready to let the AP's out of the pen. */ Modified: head/sys/x86/xen/hvm.c ============================================================================== --- head/sys/x86/xen/hvm.c Tue Mar 11 07:33:33 2014 (r263000) +++ head/sys/x86/xen/hvm.c Tue Mar 11 10:03:29 2014 (r263001) @@ -77,22 +77,12 @@ static void xen_hvm_cpu_resume(void); static void xen_hvm_cpu_init(void); /*---------------------------- Extern Declarations ---------------------------*/ -/* Variables used by mp_machdep to perform the MMU related IPIs */ -extern volatile int smp_tlb_wait; -extern vm_offset_t smp_tlb_addr2; -#ifdef __i386__ -extern vm_offset_t smp_tlb_addr1; -#else -extern struct invpcid_descr smp_tlb_invpcid; -extern uint64_t pcid_cr3; -extern int invpcid_works; -extern int pmap_pcid_enabled; -extern pmap_t smp_tlb_pmap; -#endif - #ifdef __i386__ extern void pmap_lazyfix_action(void); #endif +#ifdef __amd64__ +extern int pmap_pcid_enabled; +#endif /* Variables used by mp_machdep to perform the bitmap IPI */ extern volatile u_int cpu_ipi_pending[MAXCPU]; @@ -179,10 +169,7 @@ static int xen_smp_rendezvous_action(void *arg) { #ifdef COUNT_IPIS - int cpu; - - cpu = PCPU_GET(cpuid); - (*ipi_rendezvous_counts[cpu])++; + (*ipi_rendezvous_counts[PCPU_GET(cpuid)])++; #endif /* COUNT_IPIS */ smp_rendezvous_action(); @@ -192,20 +179,8 @@ xen_smp_rendezvous_action(void *arg) static int xen_invltlb(void *arg) { -#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS) - int cpu; - - cpu = PCPU_GET(cpuid); -#ifdef COUNT_XINVLTLB_HITS - xhits_gbl[cpu]++; -#endif /* COUNT_XINVLTLB_HITS */ -#ifdef COUNT_IPIS - (*ipi_invltlb_counts[cpu])++; -#endif /* COUNT_IPIS */ -#endif /* COUNT_XINVLTLB_HITS || COUNT_IPIS */ - invltlb(); - atomic_add_int(&smp_tlb_wait, 1); + invltlb_handler(); return (FILTER_HANDLED); } @@ -213,40 +188,8 @@ xen_invltlb(void *arg) static int xen_invltlb_pcid(void *arg) { - uint64_t cr3; -#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS) - int cpu; - - cpu = PCPU_GET(cpuid); -#ifdef COUNT_XINVLTLB_HITS - xhits_gbl[cpu]++; -#endif /* COUNT_XINVLTLB_HITS */ -#ifdef COUNT_IPIS - (*ipi_invltlb_counts[cpu])++; -#endif /* COUNT_IPIS */ -#endif /* COUNT_XINVLTLB_HITS || COUNT_IPIS */ - cr3 = rcr3(); - if (smp_tlb_invpcid.pcid != (uint64_t)-1 && - smp_tlb_invpcid.pcid != 0) { - - if (invpcid_works) { - invpcid(&smp_tlb_invpcid, INVPCID_CTX); - } else { - /* Otherwise reload %cr3 twice. */ - if (cr3 != pcid_cr3) { - load_cr3(pcid_cr3); - cr3 |= CR3_PCID_SAVE; - } - load_cr3(cr3); - } - } else { - invltlb_globpcid(); - } - if (smp_tlb_pmap != NULL) - CPU_CLR_ATOMIC(PCPU_GET(cpuid), &smp_tlb_pmap->pm_save); - - atomic_add_int(&smp_tlb_wait, 1); + invltlb_pcid_handler(); return (FILTER_HANDLED); } #endif @@ -254,24 +197,8 @@ xen_invltlb_pcid(void *arg) static int xen_invlpg(void *arg) { -#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS) - int cpu; - - cpu = PCPU_GET(cpuid); -#ifdef COUNT_XINVLTLB_HITS - xhits_pg[cpu]++; -#endif /* COUNT_XINVLTLB_HITS */ -#ifdef COUNT_IPIS - (*ipi_invlpg_counts[cpu])++; -#endif /* COUNT_IPIS */ -#endif /* COUNT_XINVLTLB_HITS || COUNT_IPIS */ -#ifdef __i386__ - invlpg(smp_tlb_addr1); -#else - invlpg(smp_tlb_invpcid.addr); -#endif - atomic_add_int(&smp_tlb_wait, 1); + invlpg_handler(); return (FILTER_HANDLED); } @@ -279,125 +206,25 @@ xen_invlpg(void *arg) static int xen_invlpg_pcid(void *arg) { -#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS) - int cpu; - - cpu = PCPU_GET(cpuid); -#ifdef COUNT_XINVLTLB_HITS - xhits_pg[cpu]++; -#endif /* COUNT_XINVLTLB_HITS */ -#ifdef COUNT_IPIS - (*ipi_invlpg_counts[cpu])++; -#endif /* COUNT_IPIS */ -#endif /* COUNT_XINVLTLB_HITS || COUNT_IPIS */ - - if (invpcid_works) { - invpcid(&smp_tlb_invpcid, INVPCID_ADDR); - } else if (smp_tlb_invpcid.pcid == 0) { - invlpg(smp_tlb_invpcid.addr); - } else if (smp_tlb_invpcid.pcid == (uint64_t)-1) { - invltlb_globpcid(); - } else { - uint64_t cr3; - - /* - * PCID supported, but INVPCID is not. - * Temporarily switch to the target address - * space and do INVLPG. - */ - cr3 = rcr3(); - if (cr3 != pcid_cr3) - load_cr3(pcid_cr3 | CR3_PCID_SAVE); - invlpg(smp_tlb_invpcid.addr); - load_cr3(cr3 | CR3_PCID_SAVE); - } - atomic_add_int(&smp_tlb_wait, 1); + invlpg_pcid_handler(); return (FILTER_HANDLED); } #endif -static inline void -invlpg_range(vm_offset_t start, vm_offset_t end) -{ - do { - invlpg(start); - start += PAGE_SIZE; - } while (start < end); -} - static int xen_invlrng(void *arg) { - vm_offset_t addr; -#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS) *** DIFF OUTPUT TRUNCATED AT 1000 LINES ***