Date: Wed, 4 Feb 2009 21:28:35 GMT From: John Baldwin <jhb@FreeBSD.org> To: Perforce Change Reviews <perforce@freebsd.org> Subject: PERFORCE change 157172 for review Message-ID: <200902042128.n14LSZAv005710@repoman.freebsd.org>
next in thread | raw e-mail | index | archive | help
http://perforce.freebsd.org/chv.cgi?CH=157172 Change 157172 by jhb@jhb_jhbbsd on 2009/02/04 21:27:39 IFC @157170 Affected files ... .. //depot/projects/smpng/sys/amd64/amd64/cpu_switch.S#23 integrate .. //depot/projects/smpng/sys/amd64/amd64/exception.S#22 integrate .. //depot/projects/smpng/sys/amd64/amd64/io_apic.c#23 integrate .. //depot/projects/smpng/sys/amd64/amd64/local_apic.c#38 integrate .. //depot/projects/smpng/sys/amd64/amd64/machdep.c#73 integrate .. //depot/projects/smpng/sys/amd64/amd64/mp_machdep.c#50 integrate .. //depot/projects/smpng/sys/amd64/amd64/msi.c#10 integrate .. //depot/projects/smpng/sys/amd64/ia32/ia32_signal.c#20 integrate .. //depot/projects/smpng/sys/amd64/ia32/ia32_sigtramp.S#5 integrate .. //depot/projects/smpng/sys/amd64/include/apicvar.h#20 integrate .. //depot/projects/smpng/sys/amd64/include/cpufunc.h#15 integrate .. //depot/projects/smpng/sys/amd64/include/intr_machdep.h#18 integrate .. //depot/projects/smpng/sys/amd64/linux32/linux32_locore.s#4 integrate .. //depot/projects/smpng/sys/amd64/linux32/linux32_sysvec.c#35 integrate .. //depot/projects/smpng/sys/arm/arm/busdma_machdep.c#26 integrate .. //depot/projects/smpng/sys/arm/arm/dump_machdep.c#7 integrate .. //depot/projects/smpng/sys/arm/arm/elf_trampoline.c#20 integrate .. //depot/projects/smpng/sys/arm/arm/pmap.c#47 integrate .. //depot/projects/smpng/sys/arm/arm/vm_machdep.c#29 integrate .. //depot/projects/smpng/sys/arm/conf/AVILA#12 integrate .. //depot/projects/smpng/sys/arm/conf/AVILA.hints#4 integrate .. //depot/projects/smpng/sys/arm/include/atomic.h#31 integrate .. //depot/projects/smpng/sys/arm/sa11x0/assabet_machdep.c#19 integrate .. //depot/projects/smpng/sys/arm/xscale/ixp425/avila_machdep.c#10 integrate .. //depot/projects/smpng/sys/arm/xscale/ixp425/files.ixp425#5 integrate .. //depot/projects/smpng/sys/arm/xscale/ixp425/ixp425.c#9 integrate .. //depot/projects/smpng/sys/arm/xscale/ixp425/ixp425reg.h#4 integrate .. //depot/projects/smpng/sys/boot/i386/pxeldr/pxeboot.8#5 integrate .. //depot/projects/smpng/sys/cam/scsi/scsi_pass.c#21 integrate .. //depot/projects/smpng/sys/compat/ndis/winx32_wrap.S#6 integrate .. //depot/projects/smpng/sys/dev/adb/adb_mouse.c#3 integrate .. //depot/projects/smpng/sys/dev/ae/if_ae.c#3 integrate .. //depot/projects/smpng/sys/dev/an/if_an.c#57 integrate .. //depot/projects/smpng/sys/dev/an/if_anreg.h#16 integrate .. //depot/projects/smpng/sys/dev/ata/ata-disk.c#69 integrate .. //depot/projects/smpng/sys/dev/ata/atapi-cam.c#38 integrate .. //depot/projects/smpng/sys/dev/ata/atapi-cd.c#69 integrate .. //depot/projects/smpng/sys/dev/ata/atapi-fd.c#38 integrate .. //depot/projects/smpng/sys/dev/ata/atapi-tape.c#39 integrate .. //depot/projects/smpng/sys/dev/ath/ath_hal/ah_regdomain.c#4 integrate .. //depot/projects/smpng/sys/dev/ath/ath_hal/ar5212/ar5212.h#3 integrate .. //depot/projects/smpng/sys/dev/ath/ath_hal/ar5212/ar5212_reset.c#4 integrate .. //depot/projects/smpng/sys/dev/ath/ath_hal/ar5416/ar5416_cal.c#3 integrate .. //depot/projects/smpng/sys/dev/cardbus/cardbus.c#36 integrate .. //depot/projects/smpng/sys/dev/cfi/cfi_bus_ixp4xx.c#1 branch .. //depot/projects/smpng/sys/dev/cfi/cfi_core.c#2 integrate .. //depot/projects/smpng/sys/dev/cfi/cfi_dev.c#2 integrate .. //depot/projects/smpng/sys/dev/firewire/firewire.c#46 integrate .. //depot/projects/smpng/sys/dev/firewire/fwohci.c#41 integrate .. //depot/projects/smpng/sys/dev/firewire/fwohci_pci.c#42 integrate .. //depot/projects/smpng/sys/dev/firewire/fwohcivar.h#14 integrate .. //depot/projects/smpng/sys/dev/firewire/sbp.c#45 integrate .. //depot/projects/smpng/sys/dev/ichsmb/ichsmb.c#16 integrate .. //depot/projects/smpng/sys/dev/ipmi/ipmi_acpi.c#2 integrate .. //depot/projects/smpng/sys/dev/ipmi/ipmi_smbios.c#4 integrate .. //depot/projects/smpng/sys/dev/kbdmux/kbdmux.c#11 integrate .. //depot/projects/smpng/sys/dev/mmc/mmc.c#10 integrate .. //depot/projects/smpng/sys/dev/mmc/mmcreg.h#5 integrate .. //depot/projects/smpng/sys/dev/pccbb/pccbb.c#69 integrate .. //depot/projects/smpng/sys/dev/pccbb/pccbb_pci.c#19 integrate .. //depot/projects/smpng/sys/dev/pccbb/pccbbvar.h#22 integrate .. //depot/projects/smpng/sys/dev/pci/pci_user.c#20 integrate .. //depot/projects/smpng/sys/dev/pci/pcireg.h#29 integrate .. //depot/projects/smpng/sys/dev/ppbus/ppbconf.h#8 integrate .. //depot/projects/smpng/sys/dev/sdhci/sdhci.c#5 integrate .. //depot/projects/smpng/sys/dev/smbus/smb.c#12 integrate .. //depot/projects/smpng/sys/dev/snp/snp.c#34 integrate .. //depot/projects/smpng/sys/dev/sound/pci/hda/hdac.c#25 integrate .. //depot/projects/smpng/sys/dev/usb2/bluetooth/ubtbcmfw2.c#4 integrate .. //depot/projects/smpng/sys/dev/usb2/controller/at91dci_atmelarm.c#3 integrate .. //depot/projects/smpng/sys/dev/usb2/controller/ehci2_pci.c#3 integrate .. //depot/projects/smpng/sys/dev/usb2/controller/musb2_otg_atmelarm.c#3 integrate .. //depot/projects/smpng/sys/dev/usb2/controller/ohci2_atmelarm.c#3 integrate .. //depot/projects/smpng/sys/dev/usb2/controller/ohci2_pci.c#3 integrate .. //depot/projects/smpng/sys/dev/usb2/controller/uhci2_pci.c#3 integrate .. //depot/projects/smpng/sys/dev/usb2/controller/uss820dci_atmelarm.c#3 integrate .. //depot/projects/smpng/sys/dev/usb2/core/usb2_compat_linux.c#3 integrate .. //depot/projects/smpng/sys/dev/usb2/core/usb2_error.c#3 integrate .. //depot/projects/smpng/sys/dev/usb2/core/usb2_hub.c#5 integrate .. //depot/projects/smpng/sys/dev/usb2/core/usb2_process.c#3 integrate .. //depot/projects/smpng/sys/dev/usb2/ethernet/if_aue2.c#5 integrate .. //depot/projects/smpng/sys/dev/usb2/ethernet/if_axe2.c#4 integrate .. //depot/projects/smpng/sys/dev/usb2/ethernet/if_cdce2.c#4 integrate .. //depot/projects/smpng/sys/dev/usb2/ethernet/if_cue2.c#4 integrate .. //depot/projects/smpng/sys/dev/usb2/ethernet/if_kue2.c#4 integrate .. //depot/projects/smpng/sys/dev/usb2/ethernet/if_rue2.c#4 integrate .. //depot/projects/smpng/sys/dev/usb2/ethernet/if_udav2.c#4 integrate .. //depot/projects/smpng/sys/dev/usb2/include/usb2_error.h#2 integrate .. //depot/projects/smpng/sys/dev/usb2/include/usb2_hid.h#2 integrate .. //depot/projects/smpng/sys/dev/usb2/include/usb2_mfunc.h#2 integrate .. //depot/projects/smpng/sys/dev/usb2/include/usb2_revision.h#2 integrate .. //depot/projects/smpng/sys/dev/usb2/input/uhid2.c#3 integrate .. //depot/projects/smpng/sys/dev/usb2/input/ukbd2.c#3 integrate .. //depot/projects/smpng/sys/dev/usb2/misc/udbp2.c#3 integrate .. //depot/projects/smpng/sys/dev/usb2/misc/ufm2.c#3 integrate .. //depot/projects/smpng/sys/dev/usb2/quirk/usb2_quirk.c#5 integrate .. //depot/projects/smpng/sys/dev/usb2/quirk/usb2_quirk.h#2 integrate .. //depot/projects/smpng/sys/dev/usb2/serial/u3g2.c#4 integrate .. //depot/projects/smpng/sys/dev/usb2/serial/uark2.c#3 integrate .. //depot/projects/smpng/sys/dev/usb2/serial/ubsa2.c#4 integrate .. //depot/projects/smpng/sys/dev/usb2/serial/ubser2.c#3 integrate .. //depot/projects/smpng/sys/dev/usb2/serial/uchcom2.c#3 integrate .. //depot/projects/smpng/sys/dev/usb2/serial/ucycom2.c#3 integrate .. //depot/projects/smpng/sys/dev/usb2/serial/ufoma2.c#4 integrate .. //depot/projects/smpng/sys/dev/usb2/serial/uftdi2.c#3 integrate .. //depot/projects/smpng/sys/dev/usb2/serial/ugensa2.c#4 integrate .. //depot/projects/smpng/sys/dev/usb2/serial/uipaq2.c#3 integrate .. //depot/projects/smpng/sys/dev/usb2/serial/umct2.c#3 integrate .. //depot/projects/smpng/sys/dev/usb2/serial/umodem2.c#3 integrate .. //depot/projects/smpng/sys/dev/usb2/serial/umoscom2.c#3 integrate .. //depot/projects/smpng/sys/dev/usb2/serial/uplcom2.c#3 integrate .. //depot/projects/smpng/sys/dev/usb2/serial/uvisor2.c#3 integrate .. //depot/projects/smpng/sys/dev/usb2/serial/uvscom2.c#3 integrate .. //depot/projects/smpng/sys/dev/usb2/sound/uaudio2.c#4 integrate .. //depot/projects/smpng/sys/dev/usb2/storage/ata-usb2.c#3 integrate .. //depot/projects/smpng/sys/dev/usb2/storage/umass2.c#5 integrate .. //depot/projects/smpng/sys/dev/usb2/storage/urio2.c#4 integrate .. //depot/projects/smpng/sys/dev/usb2/storage/ustorage2_fs.c#3 integrate .. //depot/projects/smpng/sys/dev/usb2/wlan/if_rum2.c#4 integrate .. //depot/projects/smpng/sys/dev/usb2/wlan/if_ural2.c#4 integrate .. //depot/projects/smpng/sys/dev/usb2/wlan/if_zyd2.c#4 integrate .. //depot/projects/smpng/sys/dev/xen/netback/netback.c#2 integrate .. //depot/projects/smpng/sys/dev/xen/netfront/netfront.c#5 integrate .. //depot/projects/smpng/sys/fs/coda/coda_vfsops.c#5 integrate .. //depot/projects/smpng/sys/fs/devfs/devfs_vnops.c#75 integrate .. //depot/projects/smpng/sys/fs/hpfs/hpfs_vfsops.c#39 integrate .. //depot/projects/smpng/sys/fs/nullfs/null_vnops.c#38 integrate .. //depot/projects/smpng/sys/fs/nwfs/nwfs_subr.c#12 integrate .. //depot/projects/smpng/sys/fs/nwfs/nwfs_vnops.c#21 integrate .. //depot/projects/smpng/sys/fs/smbfs/smbfs_vnops.c#43 integrate .. //depot/projects/smpng/sys/fs/tmpfs/tmpfs_subr.c#7 integrate .. //depot/projects/smpng/sys/fs/udf/udf_vnops.c#44 integrate .. //depot/projects/smpng/sys/geom/geom.h#52 integrate .. //depot/projects/smpng/sys/geom/geom_subr.c#56 integrate .. //depot/projects/smpng/sys/i386/i386/io_apic.c#25 integrate .. //depot/projects/smpng/sys/i386/i386/local_apic.c#60 integrate .. //depot/projects/smpng/sys/i386/i386/locore.s#30 integrate .. //depot/projects/smpng/sys/i386/i386/mp_machdep.c#116 integrate .. //depot/projects/smpng/sys/i386/i386/msi.c#10 integrate .. //depot/projects/smpng/sys/i386/i386/swtch.s#32 integrate .. //depot/projects/smpng/sys/i386/include/apicvar.h#31 integrate .. //depot/projects/smpng/sys/i386/include/cpufunc.h#38 integrate .. //depot/projects/smpng/sys/i386/include/intr_machdep.h#20 integrate .. //depot/projects/smpng/sys/i386/linux/linux_locore.s#3 integrate .. //depot/projects/smpng/sys/i386/svr4/svr4_locore.s#3 integrate .. //depot/projects/smpng/sys/i386/xen/mp_machdep.c#7 integrate .. //depot/projects/smpng/sys/i386/xen/xen_machdep.c#8 integrate .. //depot/projects/smpng/sys/ia64/ia64/mca.c#11 integrate .. //depot/projects/smpng/sys/ia64/include/mca.h#5 integrate .. //depot/projects/smpng/sys/kern/subr_autoconf.c#14 integrate .. //depot/projects/smpng/sys/kern/subr_bus.c#72 integrate .. //depot/projects/smpng/sys/kern/subr_clist.c#4 integrate .. //depot/projects/smpng/sys/kern/subr_devstat.c#17 integrate .. //depot/projects/smpng/sys/kern/subr_disk.c#30 integrate .. //depot/projects/smpng/sys/kern/subr_firmware.c#21 integrate .. //depot/projects/smpng/sys/kern/subr_kobj.c#9 integrate .. //depot/projects/smpng/sys/kern/subr_prf.c#56 integrate .. //depot/projects/smpng/sys/kern/subr_prof.c#32 integrate .. //depot/projects/smpng/sys/kern/subr_rman.c#39 integrate .. //depot/projects/smpng/sys/kern/subr_rtc.c#3 integrate .. //depot/projects/smpng/sys/kern/subr_taskqueue.c#37 integrate .. //depot/projects/smpng/sys/kern/subr_witness.c#177 integrate .. //depot/projects/smpng/sys/kern/sys_generic.c#56 integrate .. //depot/projects/smpng/sys/kern/tty.c#84 integrate .. //depot/projects/smpng/sys/kern/tty_info.c#3 integrate .. //depot/projects/smpng/sys/kern/tty_inq.c#3 integrate .. //depot/projects/smpng/sys/kern/tty_outq.c#3 integrate .. //depot/projects/smpng/sys/kern/uipc_socket.c#119 integrate .. //depot/projects/smpng/sys/net/if_ethersubr.c#95 integrate .. //depot/projects/smpng/sys/net/if_llatbl.c#2 integrate .. //depot/projects/smpng/sys/net80211/ieee80211.c#34 integrate .. //depot/projects/smpng/sys/net80211/ieee80211_node.c#39 integrate .. //depot/projects/smpng/sys/net80211/ieee80211_scan_sta.c#7 integrate .. //depot/projects/smpng/sys/net80211/ieee80211_tdma.c#2 integrate .. //depot/projects/smpng/sys/netatalk/ddp_usrreq.c#32 integrate .. //depot/projects/smpng/sys/netinet/ip_divert.c#62 integrate .. //depot/projects/smpng/sys/netinet/ip_ipsec.c#11 integrate .. //depot/projects/smpng/sys/netinet/ip_output.c#101 integrate .. //depot/projects/smpng/sys/netinet/sctp_auth.c#15 integrate .. //depot/projects/smpng/sys/netinet/sctp_constants.h#18 integrate .. //depot/projects/smpng/sys/netinet/sctp_crc32.c#7 integrate .. //depot/projects/smpng/sys/netinet/sctp_crc32.h#5 integrate .. //depot/projects/smpng/sys/netinet/sctp_input.c#21 integrate .. //depot/projects/smpng/sys/netinet/sctp_os_bsd.h#19 integrate .. //depot/projects/smpng/sys/netinet/sctp_output.c#24 integrate .. //depot/projects/smpng/sys/netinet/sctp_pcb.c#25 integrate .. //depot/projects/smpng/sys/netinet/sctp_pcb.h#17 integrate .. //depot/projects/smpng/sys/netinet/sctp_sysctl.c#10 integrate .. //depot/projects/smpng/sys/netinet/sctp_uio.h#16 integrate .. //depot/projects/smpng/sys/netinet/sctp_usrreq.c#22 integrate .. //depot/projects/smpng/sys/netinet/sctputil.c#25 integrate .. //depot/projects/smpng/sys/netinet/sctputil.h#14 integrate .. //depot/projects/smpng/sys/netinet6/frag6.c#21 integrate .. //depot/projects/smpng/sys/netinet6/in6.c#49 integrate .. //depot/projects/smpng/sys/netinet6/in6_gif.c#21 integrate .. //depot/projects/smpng/sys/netinet6/in6_ifattach.c#29 integrate .. //depot/projects/smpng/sys/netinet6/ip6_forward.c#26 integrate .. //depot/projects/smpng/sys/netinet6/ip6_input.c#57 integrate .. //depot/projects/smpng/sys/netinet6/ip6_var.h#27 integrate .. //depot/projects/smpng/sys/netinet6/nd6.c#49 integrate .. //depot/projects/smpng/sys/netinet6/nd6_nbr.c#33 integrate .. //depot/projects/smpng/sys/netinet6/nd6_rtr.c#27 integrate .. //depot/projects/smpng/sys/netinet6/sctp6_usrreq.c#17 integrate .. //depot/projects/smpng/sys/netinet6/vinet6.h#5 integrate .. //depot/projects/smpng/sys/netipsec/ipsec_output.c#18 integrate .. //depot/projects/smpng/sys/security/audit/audit_pipe.c#14 integrate .. //depot/projects/smpng/sys/sys/_null.h#6 integrate .. //depot/projects/smpng/sys/sys/cdefs.h#38 integrate .. //depot/projects/smpng/sys/sys/clist.h#5 integrate .. //depot/projects/smpng/sys/sys/conf.h#57 integrate .. //depot/projects/smpng/sys/sys/elf_common.h#14 integrate .. //depot/projects/smpng/sys/sys/kobj.h#7 integrate .. //depot/projects/smpng/sys/sys/malloc.h#28 integrate .. //depot/projects/smpng/sys/sys/pciio.h#4 integrate .. //depot/projects/smpng/sys/sys/sysctl.h#54 integrate .. //depot/projects/smpng/sys/sys/ttyqueue.h#4 integrate .. //depot/projects/smpng/sys/sys/unistd.h#17 integrate .. //depot/projects/smpng/sys/ufs/ffs/ffs_vfsops.c#108 integrate Differences ... ==== //depot/projects/smpng/sys/amd64/amd64/cpu_switch.S#23 (text+ko) ==== @@ -30,7 +30,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $FreeBSD: src/sys/amd64/amd64/cpu_switch.S,v 1.165 2009/01/20 12:07:49 kib Exp $ + * $FreeBSD: src/sys/amd64/amd64/cpu_switch.S,v 1.166 2009/01/31 11:37:21 obrien Exp $ */ #include <machine/asmacros.h> @@ -260,12 +260,12 @@ jmp do_tss store_seg: - movl %gs,PCB_GS(%r8) + mov %gs,PCB_GS(%r8) testl $PCB_GS32BIT,PCB_FLAGS(%r8) jnz 2f -1: movl %ds,PCB_DS(%r8) - movl %es,PCB_ES(%r8) - movl %fs,PCB_FS(%r8) +1: mov %ds,PCB_DS(%r8) + mov %es,PCB_ES(%r8) + mov %fs,PCB_FS(%r8) jmp done_store_seg 2: movq PCPU(GS32P),%rax movq (%rax),%rax @@ -277,11 +277,11 @@ jnz 2f 1: movl $MSR_GSBASE,%ecx rdmsr - movl PCB_GS(%r8),%gs + mov PCB_GS(%r8),%gs wrmsr - movl PCB_DS(%r8),%ds - movl PCB_ES(%r8),%es - movl PCB_FS(%r8),%fs + mov PCB_DS(%r8),%ds + mov PCB_ES(%r8),%es + mov PCB_FS(%r8),%fs jmp restore_fsbase /* Restore userland %gs while preserving kernel gsbase */ 2: movq PCPU(GS32P),%rax ==== //depot/projects/smpng/sys/amd64/amd64/exception.S#22 (text+ko) ==== @@ -31,7 +31,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $FreeBSD: src/sys/amd64/amd64/exception.S,v 1.136 2009/01/14 14:20:08 kib Exp $ + * $FreeBSD: src/sys/amd64/amd64/exception.S,v 1.137 2009/02/03 09:01:45 jkoshy Exp $ */ #include "opt_atpic.h" @@ -383,22 +383,24 @@ * NMI handling is special. * * First, NMIs do not respect the state of the processor's RFLAGS.IF - * bit and the NMI handler may be invoked at any time, including when - * the processor is in a critical section with RFLAGS.IF == 0. In - * particular, this means that the processor's GS.base values could be - * inconsistent on entry to the handler, and so we need to read - * MSR_GSBASE to determine if a 'swapgs' is needed. We use '%ebx', a - * C-preserved register, to remember whether to swap GS back on the - * exit path. + * bit. The NMI handler may be entered at any time, including when + * the processor is in a critical section with RFLAGS.IF == 0. + * The processor's GS.base value could be invalid on entry to the + * handler. * * Second, the processor treats NMIs specially, blocking further NMIs - * until an 'iretq' instruction is executed. We therefore need to - * execute the NMI handler with interrupts disabled to prevent a - * nested interrupt from executing an 'iretq' instruction and - * inadvertently taking the processor out of NMI mode. + * until an 'iretq' instruction is executed. We thus need to execute + * the NMI handler with interrupts disabled, to prevent a nested interrupt + * from executing an 'iretq' instruction and inadvertently taking the + * processor out of NMI mode. * - * Third, the NMI handler runs on its own stack (tss_ist1), shared - * with the double fault handler. + * Third, the NMI handler runs on its own stack (tss_ist2). The canonical + * GS.base value for the processor is stored just above the bottom of its + * NMI stack. For NMIs taken from kernel mode, the current value in + * the processor's GS.base is saved at entry to C-preserved register %r12, + * the canonical value for GS.base is then loaded into the processor, and + * the saved value is restored at exit time. For NMIs taken from user mode, + * the cheaper 'SWAPGS' instructions are used for swapping GS.base. */ IDTVEC(nmi) @@ -423,12 +425,22 @@ movq %r15,TF_R15(%rsp) xorl %ebx,%ebx testb $SEL_RPL_MASK,TF_CS(%rsp) - jnz nmi_needswapgs /* we came from userland */ + jnz nmi_fromuserspace + /* + * We've interrupted the kernel. Preserve GS.base in %r12. + */ movl $MSR_GSBASE,%ecx rdmsr - cmpl $VM_MAXUSER_ADDRESS >> 32,%edx - jae nmi_calltrap /* GS.base holds a kernel VA */ -nmi_needswapgs: + movq %rax,%r12 + shlq $32,%rdx + orq %rdx,%r12 + /* Retrieve and load the canonical value for GS.base. */ + movq TF_SIZE(%rsp),%rdx + movl %edx,%eax + shrq $32,%rdx + wrmsr + jmp nmi_calltrap +nmi_fromuserspace: incl %ebx swapgs /* Note: this label is also used by ddb and gdb: */ @@ -439,14 +451,19 @@ MEXITCOUNT #ifdef HWPMC_HOOKS /* - * Check if the current trap was from user mode and if so - * whether the current thread needs a user call chain to be - * captured. We are still in NMI mode at this point. + * Capture a userspace callchain if needed. + * + * - Check if the current trap was from user mode. + * - Check if the current thread is valid. + * - Check if the thread requires a user call chain to be + * captured. + * + * We are still in NMI mode at this point. */ - testb $SEL_RPL_MASK,TF_CS(%rsp) - jz nocallchain - movq PCPU(CURTHREAD),%rax /* curthread present? */ - orq %rax,%rax + testl %ebx,%ebx + jz nocallchain /* not from userspace */ + movq PCPU(CURTHREAD),%rax + orq %rax,%rax /* curthread present? */ jz nocallchain testl $TDP_CALLCHAIN,TD_PFLAGS(%rax) /* flagged for capture? */ jz nocallchain @@ -498,8 +515,18 @@ nocallchain: #endif testl %ebx,%ebx - jz nmi_restoreregs + jz nmi_kernelexit swapgs + jmp nmi_restoreregs +nmi_kernelexit: + /* + * Put back the preserved MSR_GSBASE value. + */ + movl $MSR_GSBASE,%ecx + movq %r12,%rdx + movl %edx,%eax + shrq $32,%rdx + wrmsr nmi_restoreregs: movq TF_RDI(%rsp),%rdi movq TF_RSI(%rsp),%rsi ==== //depot/projects/smpng/sys/amd64/amd64/io_apic.c#23 (text+ko) ==== @@ -28,7 +28,7 @@ */ #include <sys/cdefs.h> -__FBSDID("$FreeBSD: src/sys/amd64/amd64/io_apic.c,v 1.31 2007/06/05 18:57:48 jhb Exp $"); +__FBSDID("$FreeBSD: src/sys/amd64/amd64/io_apic.c,v 1.32 2009/01/29 09:22:56 jeff Exp $"); #include "opt_isa.h" @@ -327,39 +327,56 @@ { struct ioapic_intsrc *intpin = (struct ioapic_intsrc *)isrc; struct ioapic *io = (struct ioapic *)isrc->is_pic; + u_int old_vector; + u_int old_id; + + /* + * keep 1st core as the destination for NMI + */ + if (intpin->io_irq == IRQ_NMI) + apic_id = 0; + + /* + * Set us up to free the old irq. + */ + old_vector = intpin->io_vector; + old_id = intpin->io_cpu; + if (old_vector && apic_id == old_id) + return; + /* + * Allocate an APIC vector for this interrupt pin. Once + * we have a vector we program the interrupt pin. + */ intpin->io_cpu = apic_id; + intpin->io_vector = apic_alloc_vector(apic_id, intpin->io_irq); if (bootverbose) { - printf("ioapic%u: Assigning ", io->io_id); + printf("ioapic%u: routing intpin %u (", io->io_id, + intpin->io_intpin); ioapic_print_irq(intpin); - printf(" to local APIC %u\n", intpin->io_cpu); + printf(") to lapic %u vector %u\n", intpin->io_cpu, + intpin->io_vector); } ioapic_program_intpin(intpin); + /* + * Free the old vector after the new one is established. This is done + * to prevent races where we could miss an interrupt. + */ + if (old_vector) + apic_free_vector(old_id, old_vector, intpin->io_irq); } static void ioapic_enable_intr(struct intsrc *isrc) { struct ioapic_intsrc *intpin = (struct ioapic_intsrc *)isrc; - struct ioapic *io = (struct ioapic *)isrc->is_pic; - if (intpin->io_vector == 0) { - /* - * Allocate an APIC vector for this interrupt pin. Once - * we have a vector we program the interrupt pin. - */ - intpin->io_vector = apic_alloc_vector(intpin->io_irq); - if (bootverbose) { - printf("ioapic%u: routing intpin %u (", io->io_id, - intpin->io_intpin); - ioapic_print_irq(intpin); - printf(") to vector %u\n", intpin->io_vector); - } - ioapic_program_intpin(intpin); - apic_enable_vector(intpin->io_vector); - } + if (intpin->io_vector == 0) + ioapic_assign_cpu(isrc, pcpu_find(0)->pc_apic_id); + apic_enable_vector(intpin->io_cpu, intpin->io_vector); } + static void ioapic_disable_intr(struct intsrc *isrc) { @@ -369,11 +386,11 @@ if (intpin->io_vector != 0) { /* Mask this interrupt pin and free its APIC vector. */ vector = intpin->io_vector; - apic_disable_vector(vector); + apic_disable_vector(intpin->io_cpu, vector); intpin->io_masked = 1; intpin->io_vector = 0; ioapic_program_intpin(intpin); - apic_free_vector(vector, intpin->io_irq); + apic_free_vector(intpin->io_cpu, vector, intpin->io_irq); } } ==== //depot/projects/smpng/sys/amd64/amd64/local_apic.c#38 (text+ko) ==== @@ -32,7 +32,7 @@ */ #include <sys/cdefs.h> -__FBSDID("$FreeBSD: src/sys/amd64/amd64/local_apic.c,v 1.50 2008/12/11 15:56:30 jhb Exp $"); +__FBSDID("$FreeBSD: src/sys/amd64/amd64/local_apic.c,v 1.51 2009/01/29 09:22:56 jeff Exp $"); #include "opt_hwpmc_hooks.h" #include "opt_kdtrace.h" @@ -46,6 +46,8 @@ #include <sys/lock.h> #include <sys/mutex.h> #include <sys/pcpu.h> +#include <sys/proc.h> +#include <sys/sched.h> #include <sys/smp.h> #include <vm/vm.h> @@ -109,6 +111,8 @@ u_long la_hard_ticks; u_long la_stat_ticks; u_long la_prof_ticks; + /* Include IDT_SYSCALL to make indexing easier. */ + u_int la_ioint_irqs[APIC_NUM_IOINTS + 1]; } static lapics[MAX_APIC_ID + 1]; /* XXX: should thermal be an NMI? */ @@ -134,8 +138,6 @@ IDTVEC(apic_isr7), /* 224 - 255 */ }; -/* Include IDT_SYSCALL to make indexing easier. */ -static u_int ioint_irqs[APIC_NUM_IOINTS + 1]; static u_int32_t lapic_timer_divisors[] = { APIC_TDCR_1, APIC_TDCR_2, APIC_TDCR_4, APIC_TDCR_8, APIC_TDCR_16, @@ -215,14 +217,12 @@ /* Perform basic initialization of the BSP's local APIC. */ lapic_enable(); - ioint_irqs[IDT_SYSCALL - APIC_IO_INTS] = IRQ_SYSCALL; /* Set BSP's per-CPU local APIC ID. */ PCPU_SET(apic_id, lapic_id()); /* Local APIC timer interrupt. */ setidt(APIC_TIMER_INT, IDTVEC(timerint), SDT_SYSIGT, SEL_KPL, 0); - ioint_irqs[APIC_TIMER_INT - APIC_IO_INTS] = IRQ_TIMER; /* XXX: error/thermal interrupts */ } @@ -254,6 +254,9 @@ lapics[apic_id].la_lvts[i] = lvts[i]; lapics[apic_id].la_lvts[i].lvt_active = 0; } + lapics[apic_id].la_ioint_irqs[IDT_SYSCALL - APIC_IO_INTS] = IRQ_SYSCALL; + lapics[apic_id].la_ioint_irqs[APIC_TIMER_INT - APIC_IO_INTS] = + IRQ_TIMER; #ifdef SMP cpu_add(apic_id, boot_cpu); @@ -664,7 +667,8 @@ if (vector == -1) panic("Couldn't get vector from ISR!"); - isrc = intr_lookup_source(apic_idt_to_irq(vector)); + isrc = intr_lookup_source(apic_idt_to_irq(PCPU_GET(apic_id), + vector)); intr_execute_handlers(isrc, frame); } @@ -779,9 +783,19 @@ lapic->lvt_timer = value; } +u_int +apic_cpuid(u_int apic_id) +{ +#ifdef SMP + return apic_cpuids[apic_id]; +#else + return 0; +#endif +} + /* Request a free IDT vector to be used by the specified IRQ. */ u_int -apic_alloc_vector(u_int irq) +apic_alloc_vector(u_int apic_id, u_int irq) { u_int vector; @@ -793,9 +807,9 @@ */ mtx_lock_spin(&icu_lock); for (vector = 0; vector < APIC_NUM_IOINTS; vector++) { - if (ioint_irqs[vector] != 0) + if (lapics[apic_id].la_ioint_irqs[vector] != 0) continue; - ioint_irqs[vector] = irq; + lapics[apic_id].la_ioint_irqs[vector] = irq; mtx_unlock_spin(&icu_lock); return (vector + APIC_IO_INTS); } @@ -810,7 +824,7 @@ * satisfied, 0 is returned. */ u_int -apic_alloc_vectors(u_int *irqs, u_int count, u_int align) +apic_alloc_vectors(u_int apic_id, u_int *irqs, u_int count, u_int align) { u_int first, run, vector; @@ -833,7 +847,7 @@ for (vector = 0; vector < APIC_NUM_IOINTS; vector++) { /* Vector is in use, end run. */ - if (ioint_irqs[vector] != 0) { + if (lapics[apic_id].la_ioint_irqs[vector] != 0) { run = 0; first = 0; continue; @@ -853,7 +867,8 @@ /* Found a run, assign IRQs and return the first vector. */ for (vector = 0; vector < count; vector++) - ioint_irqs[first + vector] = irqs[vector]; + lapics[apic_id].la_ioint_irqs[first + vector] = + irqs[vector]; mtx_unlock_spin(&icu_lock); return (first + APIC_IO_INTS); } @@ -862,8 +877,14 @@ return (0); } +/* + * Enable a vector for a particular apic_id. Since all lapics share idt + * entries and ioint_handlers this enables the vector on all lapics. lapics + * which do not have the vector configured would report spurious interrupts + * should it fire. + */ void -apic_enable_vector(u_int vector) +apic_enable_vector(u_int apic_id, u_int vector) { KASSERT(vector != IDT_SYSCALL, ("Attempt to overwrite syscall entry")); @@ -873,7 +894,7 @@ } void -apic_disable_vector(u_int vector) +apic_disable_vector(u_int apic_id, u_int vector) { KASSERT(vector != IDT_SYSCALL, ("Attempt to overwrite syscall entry")); @@ -884,27 +905,42 @@ /* Release an APIC vector when it's no longer in use. */ void -apic_free_vector(u_int vector, u_int irq) +apic_free_vector(u_int apic_id, u_int vector, u_int irq) { + struct thread *td; KASSERT(vector >= APIC_IO_INTS && vector != IDT_SYSCALL && vector <= APIC_IO_INTS + APIC_NUM_IOINTS, ("Vector %u does not map to an IRQ line", vector)); KASSERT(irq < NUM_IO_INTS, ("Invalid IRQ %u", irq)); - KASSERT(ioint_irqs[vector - APIC_IO_INTS] == irq, ("IRQ mismatch")); + KASSERT(lapics[apic_id].la_ioint_irqs[vector - APIC_IO_INTS] == + irq, ("IRQ mismatch")); + + /* + * Bind us to the cpu that owned the vector before freeing it so + * we don't lose an interrupt delivery race. + */ + td = curthread; + thread_lock(td); + if (sched_is_bound(td)) + panic("apic_free_vector: Thread already bound.\n"); + sched_bind(td, apic_cpuid(apic_id)); mtx_lock_spin(&icu_lock); - ioint_irqs[vector - APIC_IO_INTS] = 0; + lapics[apic_id].la_ioint_irqs[vector - APIC_IO_INTS] = 0; mtx_unlock_spin(&icu_lock); + sched_unbind(td); + thread_unlock(td); + } /* Map an IDT vector (APIC) to an IRQ (interrupt source). */ u_int -apic_idt_to_irq(u_int vector) +apic_idt_to_irq(u_int apic_id, u_int vector) { KASSERT(vector >= APIC_IO_INTS && vector != IDT_SYSCALL && vector <= APIC_IO_INTS + APIC_NUM_IOINTS, ("Vector %u does not map to an IRQ line", vector)); - return (ioint_irqs[vector - APIC_IO_INTS]); + return (lapics[apic_id].la_ioint_irqs[vector - APIC_IO_INTS]); } #ifdef DDB @@ -915,6 +951,7 @@ { struct intsrc *isrc; int i, verbose; + u_int apic_id; u_int irq; if (strcmp(modif, "vv") == 0) @@ -923,9 +960,14 @@ verbose = 1; else verbose = 0; - for (i = 0; i < APIC_NUM_IOINTS + 1 && !db_pager_quit; i++) { - irq = ioint_irqs[i]; - if (irq != 0 && irq != IRQ_SYSCALL) { + for (apic_id = 0; apic_id <= MAX_APIC_ID; apic_id++) { + if (lapics[apic_id].la_present == 0) + continue; + db_printf("Interrupts bound to lapic %u\n", apic_id); + for (i = 0; i < APIC_NUM_IOINTS + 1 && !db_pager_quit; i++) { + irq = lapics[apic_id].la_ioint_irqs[i]; + if (irq == 0 || irq == IRQ_SYSCALL) + continue; db_printf("vec 0x%2x -> ", i + APIC_IO_INTS); if (irq == IRQ_TIMER) db_printf("lapic timer\n"); ==== //depot/projects/smpng/sys/amd64/amd64/machdep.c#73 (text+ko) ==== @@ -39,7 +39,7 @@ */ #include <sys/cdefs.h> -__FBSDID("$FreeBSD: src/sys/amd64/amd64/machdep.c,v 1.690 2008/09/08 09:59:05 kib Exp $"); +__FBSDID("$FreeBSD: src/sys/amd64/amd64/machdep.c,v 1.691 2009/02/03 09:01:45 jkoshy Exp $"); #include "opt_atalk.h" #include "opt_atpic.h" @@ -809,6 +809,9 @@ static char dblfault_stack[PAGE_SIZE] __aligned(16); +static char nmi0_stack[PAGE_SIZE] __aligned(16); +CTASSERT(sizeof(struct nmi_pcpu) == 16); + struct amd64tss common_tss[MAXCPU]; /* software prototypes -- in more palatable form */ @@ -1291,6 +1294,7 @@ caddr_t kmdp; int gsel_tss, x; struct pcpu *pc; + struct nmi_pcpu *np; u_int64_t msr; char *env; @@ -1365,7 +1369,7 @@ setidt(x, &IDTVEC(rsvd), SDT_SYSIGT, SEL_KPL, 0); setidt(IDT_DE, &IDTVEC(div), SDT_SYSIGT, SEL_KPL, 0); setidt(IDT_DB, &IDTVEC(dbg), SDT_SYSIGT, SEL_KPL, 0); - setidt(IDT_NMI, &IDTVEC(nmi), SDT_SYSIGT, SEL_KPL, 1); + setidt(IDT_NMI, &IDTVEC(nmi), SDT_SYSIGT, SEL_KPL, 2); setidt(IDT_BP, &IDTVEC(bpt), SDT_SYSIGT, SEL_UPL, 0); setidt(IDT_OF, &IDTVEC(ofl), SDT_SYSIGT, SEL_KPL, 0); setidt(IDT_BR, &IDTVEC(bnd), SDT_SYSIGT, SEL_KPL, 0); @@ -1438,6 +1442,14 @@ /* doublefault stack space, runs on ist1 */ common_tss[0].tss_ist1 = (long)&dblfault_stack[sizeof(dblfault_stack)]; + /* + * NMI stack, runs on ist2. The pcpu pointer is stored just + * above the start of the ist2 stack. + */ + np = ((struct nmi_pcpu *) &nmi0_stack[sizeof(nmi0_stack)]) - 1; + np->np_pcpu = (register_t) pc; + common_tss[0].tss_ist2 = (long) np; + /* Set the IO permission bitmap (empty due to tss seg limit) */ common_tss[0].tss_iobase = sizeof(struct amd64tss); ==== //depot/projects/smpng/sys/amd64/amd64/mp_machdep.c#50 (text+ko) ==== @@ -25,7 +25,7 @@ */ #include <sys/cdefs.h> -__FBSDID("$FreeBSD: src/sys/amd64/amd64/mp_machdep.c,v 1.295 2008/11/26 19:25:13 jkim Exp $"); +__FBSDID("$FreeBSD: src/sys/amd64/amd64/mp_machdep.c,v 1.297 2009/02/03 09:01:45 jkoshy Exp $"); #include "opt_cpu.h" #include "opt_kstack_pages.h" @@ -92,6 +92,7 @@ /* Temporary holder for double fault stack */ char *doublefault_stack; +char *nmi_stack; /* Hotwire a 0->4MB V==P mapping */ extern pt_entry_t *KPTphys; @@ -152,6 +153,7 @@ int cpu_disabled:1; } static cpu_info[MAX_APIC_ID + 1]; int cpu_apic_ids[MAXCPU]; +int apic_cpuids[MAX_APIC_ID + 1]; /* Holds pending bitmap based IPIs per CPU */ static volatile u_int cpu_ipi_pending[MAXCPU]; @@ -349,6 +351,7 @@ KASSERT(boot_cpu_id == PCPU_GET(apic_id), ("BSP's APIC ID doesn't match boot_cpu_id")); cpu_apic_ids[0] = boot_cpu_id; + apic_cpuids[boot_cpu_id] = 0; assign_cpu_ids(); @@ -435,6 +438,7 @@ init_secondary(void) { struct pcpu *pc; + struct nmi_pcpu *np; u_int64_t msr, cr0; int cpu, gsel_tss, x; struct region_descriptor ap_gdt; @@ -448,6 +452,10 @@ common_tss[cpu].tss_iobase = sizeof(struct amd64tss); common_tss[cpu].tss_ist1 = (long)&doublefault_stack[PAGE_SIZE]; + /* The NMI stack runs on IST2. */ + np = ((struct nmi_pcpu *) &nmi_stack[PAGE_SIZE]) - 1; + common_tss[cpu].tss_ist2 = (long) np; + /* Prepare private GDT */ gdt_segs[GPROC0_SEL].ssd_base = (long) &common_tss[cpu]; ssdtosyssd(&gdt_segs[GPROC0_SEL], @@ -472,6 +480,9 @@ pc->pc_rsp0 = 0; pc->pc_gs32p = &gdt[NGDT * cpu + GUGS32_SEL]; + /* Save the per-cpu pointer for use by the NMI handler. */ + np->np_pcpu = (register_t) pc; + wrmsr(MSR_FSBASE, 0); /* User value */ wrmsr(MSR_GSBASE, (u_int64_t)pc); wrmsr(MSR_KGSBASE, (u_int64_t)pc); /* XXX User value while we're in the kernel */ @@ -656,6 +667,7 @@ if (mp_ncpus < MAXCPU) { cpu_apic_ids[mp_ncpus] = i; + apic_cpuids[i] = mp_ncpus; mp_ncpus++; } else cpu_info[i].cpu_disabled = 1; @@ -722,6 +734,7 @@ /* allocate and set up an idle stack data page */ bootstacks[cpu] = (void *)kmem_alloc(kernel_map, KSTACK_PAGES * PAGE_SIZE); doublefault_stack = (char *)kmem_alloc(kernel_map, PAGE_SIZE); + nmi_stack = (char *)kmem_alloc(kernel_map, PAGE_SIZE); bootSTK = (char *)bootstacks[cpu] + KSTACK_PAGES * PAGE_SIZE - 8; bootAP = cpu; ==== //depot/projects/smpng/sys/amd64/amd64/msi.c#10 (text+ko) ==== @@ -35,7 +35,7 @@ */ #include <sys/cdefs.h> -__FBSDID("$FreeBSD: src/sys/amd64/amd64/msi.c,v 1.9 2009/01/12 19:17:35 jkim Exp $"); +__FBSDID("$FreeBSD: src/sys/amd64/amd64/msi.c,v 1.10 2009/01/29 09:22:56 jeff Exp $"); #include <sys/param.h> #include <sys/bus.h> @@ -161,7 +161,9 @@ { struct msi_intsrc *msi = (struct msi_intsrc *)isrc; - apic_enable_vector(msi->msi_vector); + if (msi->msi_vector == 0) + msi_assign_cpu(isrc, 0); + apic_enable_vector(msi->msi_cpu, msi->msi_vector); } static void @@ -169,7 +171,7 @@ { struct msi_intsrc *msi = (struct msi_intsrc *)isrc; - apic_disable_vector(msi->msi_vector); + apic_disable_vector(msi->msi_cpu, msi->msi_vector); } static int @@ -199,15 +201,35 @@ msi_assign_cpu(struct intsrc *isrc, u_int apic_id) { struct msi_intsrc *msi = (struct msi_intsrc *)isrc; + int old_vector; + u_int old_id; + int vector; + /* Store information to free existing irq. */ + old_vector = msi->msi_vector; + old_id = msi->msi_cpu; + if (old_vector && old_id == apic_id) + return; + /* Allocate IDT vector on this cpu. */ + vector = apic_alloc_vector(apic_id, msi->msi_irq); + if (vector == 0) + return; /* XXX alloc_vector panics on failure. */ msi->msi_cpu = apic_id; + msi->msi_vector = vector; if (bootverbose) - printf("msi: Assigning %s IRQ %d to local APIC %u\n", + printf("msi: Assigning %s IRQ %d to local APIC %u vector %u\n", msi->msi_msix ? "MSI-X" : "MSI", msi->msi_irq, - msi->msi_cpu); + msi->msi_cpu, msi->msi_vector); pci_remap_msi_irq(msi->msi_dev, msi->msi_irq); + /* + * Free the old vector after the new one is established. This is done + * to prevent races where we could miss an interrupt. + */ + if (old_vector) + apic_free_vector(old_id, old_vector, msi->msi_irq); } + void msi_init(void) { @@ -263,7 +285,7 @@ msi_alloc(device_t dev, int count, int maxcount, int *irqs) { struct msi_intsrc *msi, *fsrc; - int cnt, i, vector; + int cnt, i; if (!msi_enabled) return (ENXIO); @@ -309,22 +331,12 @@ /* Ok, we now have the IRQs allocated. */ KASSERT(cnt == count, ("count mismatch")); - /* Allocate 'count' IDT vectors. */ - vector = apic_alloc_vectors(irqs, count, maxcount); - if (vector == 0) { - mtx_unlock(&msi_lock); - return (ENOSPC); - } - /* Assign IDT vectors and make these messages owned by 'dev'. */ fsrc = (struct msi_intsrc *)intr_lookup_source(irqs[0]); for (i = 0; i < count; i++) { msi = (struct msi_intsrc *)intr_lookup_source(irqs[i]); msi->msi_dev = dev; - msi->msi_vector = vector + i; - if (bootverbose) - printf("msi: routing MSI IRQ %d to vector %u\n", - msi->msi_irq, msi->msi_vector); + msi->msi_vector = 0; msi->msi_first = fsrc; KASSERT(msi->msi_intsrc.is_handlers == 0, ("dead MSI has handlers")); @@ -377,14 +389,18 @@ KASSERT(msi->msi_dev == first->msi_dev, ("owner mismatch")); msi->msi_first = NULL; msi->msi_dev = NULL; - apic_free_vector(msi->msi_vector, msi->msi_irq); + if (msi->msi_vector) + apic_free_vector(msi->msi_cpu, msi->msi_vector, + msi->msi_irq); msi->msi_vector = 0; } /* Clear out the first message. */ first->msi_first = NULL; first->msi_dev = NULL; - apic_free_vector(first->msi_vector, first->msi_irq); + if (first->msi_vector) + apic_free_vector(first->msi_cpu, first->msi_vector, + first->msi_irq); first->msi_vector = 0; first->msi_count = 0; @@ -433,7 +449,7 @@ msix_alloc(device_t dev, int *irq) { struct msi_intsrc *msi; - int i, vector; + int i; if (!msi_enabled) return (ENXIO); @@ -468,15 +484,9 @@ goto again; >>> TRUNCATED FOR MAIL (1000 lines) <<<
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?200902042128.n14LSZAv005710>