Date: Thu, 16 Jun 2011 20:58:24 +0000 (UTC) From: Andreas Tobler <andreast@FreeBSD.org> To: src-committers@freebsd.org, svn-src-projects@freebsd.org Subject: svn commit: r223161 - in projects/pseries: . amd64/acpica amd64/amd64 amd64/conf amd64/include arm/arm arm/include boot/forth boot/i386/zfsboot boot/ia64/common boot/ia64/efi cam cam/ata cam/scsi c... Message-ID: <201106162058.p5GKwODG045574@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: andreast Date: Thu Jun 16 20:58:23 2011 New Revision: 223161 URL: http://svn.freebsd.org/changeset/base/223161 Log: MFC. Added: projects/pseries/boot/ia64/common/icache.c - copied unchanged from r223156, head/sys/boot/ia64/common/icache.c projects/pseries/dev/usb/serial/umcs.c - copied unchanged from r223156, head/sys/dev/usb/serial/umcs.c projects/pseries/dev/usb/serial/umcs.h - copied unchanged from r223156, head/sys/dev/usb/serial/umcs.h projects/pseries/modules/usb/umcs/ - copied from r223156, head/sys/modules/usb/umcs/ projects/pseries/netinet/in_pcbgroup.c - copied unchanged from r223156, head/sys/netinet/in_pcbgroup.c projects/pseries/netinet6/in6_pcbgroup.c - copied unchanged from r223156, head/sys/netinet6/in6_pcbgroup.c projects/pseries/sys/_cpuset.h - copied unchanged from r223156, head/sys/sys/_cpuset.h Replaced: projects/pseries/dev/iicbus/ad7417.c - copied unchanged from r223156, head/sys/dev/iicbus/ad7417.c projects/pseries/nfs/nfs_kdtrace.h - copied unchanged from r223156, head/sys/nfs/nfs_kdtrace.h projects/pseries/sys/_stdint.h - copied unchanged from r223156, head/sys/sys/_stdint.h Deleted: projects/pseries/dev/usb/serial/umcs7840.c projects/pseries/dev/usb/serial/umcs7840.h projects/pseries/modules/usb/umcs7840/ Modified: projects/pseries/Makefile projects/pseries/amd64/acpica/acpi_wakeup.c projects/pseries/amd64/amd64/intr_machdep.c projects/pseries/amd64/amd64/legacy.c projects/pseries/amd64/amd64/machdep.c projects/pseries/amd64/amd64/mp_machdep.c projects/pseries/amd64/amd64/pmap.c projects/pseries/amd64/amd64/vm_machdep.c projects/pseries/amd64/conf/GENERIC projects/pseries/amd64/include/_types.h projects/pseries/amd64/include/pmap.h projects/pseries/amd64/include/smp.h projects/pseries/arm/arm/pmap.c projects/pseries/arm/include/_types.h projects/pseries/arm/include/pmap.h projects/pseries/boot/forth/loader.rc projects/pseries/boot/i386/zfsboot/Makefile projects/pseries/boot/ia64/common/Makefile projects/pseries/boot/ia64/common/exec.c projects/pseries/boot/ia64/common/libia64.h projects/pseries/boot/ia64/efi/efimd.c projects/pseries/boot/ia64/efi/main.c projects/pseries/boot/ia64/efi/version projects/pseries/cam/ata/ata_all.c projects/pseries/cam/ata/ata_da.c projects/pseries/cam/ata/ata_xpt.c projects/pseries/cam/cam_ccb.h projects/pseries/cam/cam_periph.c projects/pseries/cam/cam_periph.h projects/pseries/cam/cam_xpt.c projects/pseries/cam/cam_xpt.h projects/pseries/cam/cam_xpt_internal.h projects/pseries/cam/scsi/scsi_all.c projects/pseries/cam/scsi/scsi_all.h projects/pseries/cam/scsi/scsi_da.c projects/pseries/cam/scsi/scsi_pass.c projects/pseries/cam/scsi/scsi_xpt.c projects/pseries/cddl/compat/opensolaris/sys/atomic.h projects/pseries/cddl/compat/opensolaris/sys/kstat.h projects/pseries/cddl/contrib/opensolaris/uts/common/fs/zfs/spa_history.c projects/pseries/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/arc.h projects/pseries/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/ddt.h projects/pseries/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/dsl_pool.h projects/pseries/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/spa.h projects/pseries/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/zfs_ioctl.h projects/pseries/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/zio.h projects/pseries/cddl/dev/cyclic/i386/cyclic_machdep.c projects/pseries/cddl/dev/dtrace/amd64/dtrace_subr.c projects/pseries/cddl/dev/dtrace/i386/dtrace_subr.c projects/pseries/conf/Makefile.arm projects/pseries/conf/Makefile.powerpc projects/pseries/conf/NOTES projects/pseries/conf/files projects/pseries/conf/options projects/pseries/ddb/db_command.c projects/pseries/dev/aac/aac.c projects/pseries/dev/aac/aacvar.h projects/pseries/dev/acpica/acpi.c projects/pseries/dev/amdsbwd/amdsbwd.c projects/pseries/dev/ata/ata-sata.c projects/pseries/dev/ata/chipsets/ata-intel.c projects/pseries/dev/ath/ath_hal/ah.h projects/pseries/dev/ath/ath_hal/ar5212/ar5212.h projects/pseries/dev/ath/ath_hal/ar5212/ar5212_attach.c projects/pseries/dev/ath/ath_hal/ar5212/ar5212_misc.c projects/pseries/dev/ath/ath_hal/ar5416/ar5416.h projects/pseries/dev/ath/ath_hal/ar5416/ar5416_attach.c projects/pseries/dev/ath/ath_hal/ar5416/ar5416_misc.c projects/pseries/dev/ath/if_ath.c projects/pseries/dev/ath/if_ath_ahb.c projects/pseries/dev/ath/if_athvar.h projects/pseries/dev/atkbdc/atkbd.c projects/pseries/dev/atkbdc/atkbdreg.h projects/pseries/dev/bxe/bxe_debug.h projects/pseries/dev/bxe/bxe_link.c projects/pseries/dev/bxe/if_bxe.c projects/pseries/dev/bxe/if_bxe.h projects/pseries/dev/cardbus/cardbus_cis.c projects/pseries/dev/cxgbe/adapter.h projects/pseries/dev/cxgbe/t4_ioctl.h projects/pseries/dev/cxgbe/t4_main.c projects/pseries/dev/cxgbe/t4_sge.c projects/pseries/dev/hwpmc/hwpmc_mod.c projects/pseries/dev/iwn/if_iwn.c projects/pseries/dev/pccard/pccard.c projects/pseries/dev/pci/pci.c projects/pseries/dev/pci/pci_pci.c projects/pseries/dev/puc/puc.c projects/pseries/dev/puc/puc_bfe.h projects/pseries/dev/puc/puc_pccard.c projects/pseries/dev/puc/puc_pci.c projects/pseries/dev/puc/pucdata.c projects/pseries/dev/safe/safe.c projects/pseries/dev/sound/pci/hda/hdac.c projects/pseries/dev/sound/pcm/sound.c projects/pseries/dev/usb/usb_device.h projects/pseries/dev/usb/usb_freebsd.h projects/pseries/dev/usb/usb_generic.c projects/pseries/dev/usb/usb_hub.c projects/pseries/dev/usb/usb_request.c projects/pseries/dev/usb/usb_request.h projects/pseries/dev/usb/usb_transfer.c projects/pseries/dev/usb/usbdi.h projects/pseries/dev/xen/blkback/blkback.c projects/pseries/dev/xen/blkfront/blkfront.c projects/pseries/dev/xen/control/control.c projects/pseries/dev/xen/netfront/netfront.c projects/pseries/fs/nfs/nfs_var.h projects/pseries/fs/nfsclient/nfs_clport.c projects/pseries/fs/nfsclient/nfs_clrpcops.c projects/pseries/fs/nfsclient/nfs_clstate.c projects/pseries/fs/nfsclient/nfs_clvnops.c projects/pseries/geom/eli/g_eli.c projects/pseries/geom/geom.h projects/pseries/geom/geom_dev.c projects/pseries/geom/geom_disk.c projects/pseries/geom/geom_disk.h projects/pseries/geom/geom_event.c projects/pseries/geom/geom_subr.c projects/pseries/i386/conf/GENERIC projects/pseries/i386/i386/intr_machdep.c projects/pseries/i386/i386/legacy.c projects/pseries/i386/i386/machdep.c projects/pseries/i386/i386/mp_machdep.c projects/pseries/i386/i386/pmap.c projects/pseries/i386/i386/vm_machdep.c projects/pseries/i386/include/_types.h projects/pseries/i386/include/pmap.h projects/pseries/i386/include/sf_buf.h projects/pseries/i386/include/smp.h projects/pseries/i386/xen/mp_machdep.c projects/pseries/i386/xen/pmap.c projects/pseries/ia64/acpica/acpi_machdep.c projects/pseries/ia64/ia64/machdep.c projects/pseries/ia64/ia64/mp_machdep.c projects/pseries/ia64/ia64/pal.S projects/pseries/ia64/include/_types.h projects/pseries/ia64/include/smp.h projects/pseries/kern/kern_conf.c projects/pseries/kern/kern_cpuset.c projects/pseries/kern/kern_exit.c projects/pseries/kern/kern_ktr.c projects/pseries/kern/kern_pmc.c projects/pseries/kern/kern_rmlock.c projects/pseries/kern/kern_shutdown.c projects/pseries/kern/ksched.c projects/pseries/kern/link_elf.c projects/pseries/kern/sched_4bsd.c projects/pseries/kern/sched_ule.c projects/pseries/kern/subr_devstat.c projects/pseries/kern/subr_kdb.c projects/pseries/kern/subr_pcpu.c projects/pseries/kern/subr_prf.c projects/pseries/kern/subr_rman.c projects/pseries/kern/subr_smp.c projects/pseries/kern/sys_process.c projects/pseries/kern/uipc_syscalls.c projects/pseries/kern/vfs_bio.c projects/pseries/mips/cavium/octeon_mp.c projects/pseries/mips/include/_types.h projects/pseries/mips/include/hwfunc.h projects/pseries/mips/include/pmap.h projects/pseries/mips/include/smp.h projects/pseries/mips/mips/mp_machdep.c projects/pseries/mips/mips/pmap.c projects/pseries/mips/rmi/xlr_machdep.c projects/pseries/mips/sibyte/sb_scd.c projects/pseries/modules/usb/Makefile projects/pseries/net/bridgestp.c projects/pseries/net/bridgestp.h projects/pseries/net/if.h projects/pseries/net80211/ieee80211_acl.c projects/pseries/net80211/ieee80211_ioctl.c projects/pseries/net80211/ieee80211_ioctl.h projects/pseries/netgraph/ng_nat.c projects/pseries/netinet/icmp6.h projects/pseries/netinet/in_pcb.c projects/pseries/netinet/in_pcb.h projects/pseries/netinet/ip_divert.c projects/pseries/netinet/ip_input.c projects/pseries/netinet/ip_ipsec.c projects/pseries/netinet/ipfw/ip_fw2.c projects/pseries/netinet/ipfw/ip_fw_nat.c projects/pseries/netinet/ipfw/ip_fw_sockopt.c projects/pseries/netinet/libalias/alias.h projects/pseries/netinet/libalias/alias_sctp.h projects/pseries/netinet/raw_ip.c projects/pseries/netinet/sctp.h projects/pseries/netinet/sctp_auth.c projects/pseries/netinet/sctp_indata.c projects/pseries/netinet/sctp_output.c projects/pseries/netinet/sctp_pcb.c projects/pseries/netinet/sctp_structs.h projects/pseries/netinet/sctp_uio.h projects/pseries/netinet/sctp_usrreq.c projects/pseries/netinet/sctp_var.h projects/pseries/netinet/sctputil.c projects/pseries/netinet/tcp_output.c projects/pseries/netinet/tcp_subr.c projects/pseries/netinet/tcp_syncache.c projects/pseries/netinet/udp_usrreq.c projects/pseries/netinet6/in6.c projects/pseries/netinet6/in6.h projects/pseries/netinet6/in6_pcb.c projects/pseries/netinet6/in6_pcb.h projects/pseries/netinet6/in6_proto.c projects/pseries/netinet6/ip6_input.c projects/pseries/netinet6/ip6_ipsec.c projects/pseries/netinet6/ip6_var.h projects/pseries/netinet6/nd6.c projects/pseries/netinet6/nd6.h projects/pseries/netinet6/nd6_nbr.c projects/pseries/netinet6/nd6_rtr.c projects/pseries/netinet6/send.h projects/pseries/ofed/include/linux/list.h projects/pseries/pc98/conf/GENERIC projects/pseries/pc98/pc98/machdep.c projects/pseries/powerpc/aim/mmu_oea.c projects/pseries/powerpc/aim/mmu_oea64.c projects/pseries/powerpc/booke/platform_bare.c projects/pseries/powerpc/booke/pmap.c projects/pseries/powerpc/conf/GENERIC projects/pseries/powerpc/conf/GENERIC64 projects/pseries/powerpc/include/_types.h projects/pseries/powerpc/include/openpicvar.h projects/pseries/powerpc/include/pmap.h projects/pseries/powerpc/include/smp.h projects/pseries/powerpc/mpc85xx/openpic_fdt.c projects/pseries/powerpc/powerpc/intr_machdep.c projects/pseries/powerpc/powerpc/mp_machdep.c projects/pseries/powerpc/powerpc/openpic.c projects/pseries/powerpc/powerpc/pic_if.m projects/pseries/sparc64/conf/GENERIC projects/pseries/sparc64/include/_types.h projects/pseries/sparc64/include/ktr.h projects/pseries/sparc64/include/pmap.h projects/pseries/sparc64/include/smp.h projects/pseries/sparc64/sparc64/exception.S projects/pseries/sparc64/sparc64/genassym.c projects/pseries/sparc64/sparc64/intr_machdep.c projects/pseries/sparc64/sparc64/mp_exception.S projects/pseries/sparc64/sparc64/mp_locore.S projects/pseries/sparc64/sparc64/mp_machdep.c projects/pseries/sparc64/sparc64/pmap.c projects/pseries/sparc64/sparc64/swtch.S projects/pseries/sparc64/sparc64/tlb.c projects/pseries/sys/_rmlock.h projects/pseries/sys/conf.h projects/pseries/sys/cpuset.h projects/pseries/sys/disk.h projects/pseries/sys/ktr.h projects/pseries/sys/mbuf.h projects/pseries/sys/pcpu.h projects/pseries/sys/pmckern.h projects/pseries/sys/proc.h projects/pseries/sys/smp.h projects/pseries/sys/soundcard.h projects/pseries/sys/types.h projects/pseries/sys/vnode.h projects/pseries/ufs/ffs/ffs_alloc.c projects/pseries/ufs/ffs/ffs_balloc.c projects/pseries/ufs/ffs/ffs_extern.h projects/pseries/ufs/ffs/ffs_inode.c projects/pseries/ufs/ffs/ffs_snapshot.c projects/pseries/ufs/ffs/ffs_softdep.c projects/pseries/ufs/ffs/ffs_vfsops.c projects/pseries/ufs/ffs/ffs_vnops.c projects/pseries/ufs/ffs/fs.h projects/pseries/ufs/ffs/softdep.h projects/pseries/ufs/ufs/inode.h projects/pseries/ufs/ufs/quota.h projects/pseries/ufs/ufs/ufs_lookup.c projects/pseries/ufs/ufs/ufs_quota.c projects/pseries/ufs/ufs/ufs_vnops.c projects/pseries/ufs/ufs/ufsmount.h projects/pseries/vm/vm_page.c projects/pseries/vm/vm_page.h projects/pseries/vm/vnode_pager.c projects/pseries/x86/x86/local_apic.c projects/pseries/x86/x86/tsc.c projects/pseries/xen/interface/io/xenbus.h projects/pseries/xen/xenbus/xenbus.c projects/pseries/xen/xenbus/xenbus_if.m projects/pseries/xen/xenbus/xenbusb.c projects/pseries/xen/xenbus/xenbusb.h projects/pseries/xen/xenbus/xenbusb_back.c projects/pseries/xen/xenbus/xenbusb_front.c projects/pseries/xen/xenbus/xenbusb_if.m projects/pseries/xen/xenbus/xenbusvar.h projects/pseries/xen/xenstore/xenstorevar.h Directory Properties: projects/pseries/ (props changed) projects/pseries/amd64/include/xen/ (props changed) projects/pseries/boot/ (props changed) projects/pseries/boot/i386/efi/ (props changed) projects/pseries/boot/ia64/efi/ (props changed) projects/pseries/boot/ia64/ski/ (props changed) projects/pseries/boot/powerpc/boot1.chrp/ (props changed) projects/pseries/boot/powerpc/ofw/ (props changed) projects/pseries/cddl/contrib/opensolaris/ (props changed) projects/pseries/conf/ (props changed) projects/pseries/contrib/dev/acpica/ (props changed) projects/pseries/contrib/octeon-sdk/ (props changed) projects/pseries/contrib/pf/ (props changed) projects/pseries/contrib/x86emu/ (props changed) Modified: projects/pseries/Makefile ============================================================================== --- projects/pseries/Makefile Thu Jun 16 19:57:07 2011 (r223160) +++ projects/pseries/Makefile Thu Jun 16 20:58:23 2011 (r223161) @@ -12,7 +12,7 @@ CSCOPEDIRS= boot bsm cam cddl compat con geom gnu isa kern libkern modules net net80211 netatalk \ netgraph netinet netinet6 netipsec netipx netnatm netncp \ netsmb nfs nfsclient nfsserver nlm opencrypto \ - pci rpc security sys ufs vm xdr ${CSCOPE_ARCHDIR} + pci rpc security sys ufs vm xdr xen ${CSCOPE_ARCHDIR} .if defined(ALL_ARCH) CSCOPE_ARCHDIR ?= amd64 arm i386 ia64 mips pc98 powerpc sparc64 x86 .else Modified: projects/pseries/amd64/acpica/acpi_wakeup.c ============================================================================== --- projects/pseries/amd64/acpica/acpi_wakeup.c Thu Jun 16 19:57:07 2011 (r223160) +++ projects/pseries/amd64/acpica/acpi_wakeup.c Thu Jun 16 20:58:23 2011 (r223161) @@ -78,7 +78,7 @@ static void acpi_stop_beep(void *); #ifdef SMP static int acpi_wakeup_ap(struct acpi_softc *, int); -static void acpi_wakeup_cpus(struct acpi_softc *, cpumask_t); +static void acpi_wakeup_cpus(struct acpi_softc *, const cpuset_t *); #endif #define WAKECODE_VADDR(sc) ((sc)->acpi_wakeaddr + (3 * PAGE_SIZE)) @@ -173,7 +173,7 @@ acpi_wakeup_ap(struct acpi_softc *sc, in #define BIOS_WARM (0x0a) static void -acpi_wakeup_cpus(struct acpi_softc *sc, cpumask_t wakeup_cpus) +acpi_wakeup_cpus(struct acpi_softc *sc, const cpuset_t *wakeup_cpus) { uint32_t mpbioswarmvec; int cpu; @@ -192,7 +192,7 @@ acpi_wakeup_cpus(struct acpi_softc *sc, /* Wake up each AP. */ for (cpu = 1; cpu < mp_ncpus; cpu++) { - if ((wakeup_cpus & (1 << cpu)) == 0) + if (!CPU_ISSET(cpu, wakeup_cpus)) continue; if (acpi_wakeup_ap(sc, cpu) == 0) { /* restore the warmstart vector */ @@ -214,7 +214,7 @@ int acpi_sleep_machdep(struct acpi_softc *sc, int state) { #ifdef SMP - cpumask_t wakeup_cpus; + cpuset_t wakeup_cpus; #endif register_t cr3, rf; ACPI_STATUS status; @@ -244,10 +244,9 @@ acpi_sleep_machdep(struct acpi_softc *sc if (savectx(susppcbs[0])) { #ifdef SMP - if (wakeup_cpus != 0 && suspend_cpus(wakeup_cpus) == 0) { - device_printf(sc->acpi_dev, - "Failed to suspend APs: CPU mask = 0x%jx\n", - (uintmax_t)(wakeup_cpus & ~stopped_cpus)); + if (!CPU_EMPTY(&wakeup_cpus) && + suspend_cpus(wakeup_cpus) == 0) { + device_printf(sc->acpi_dev, "Failed to suspend APs\n"); goto out; } #endif @@ -282,8 +281,8 @@ acpi_sleep_machdep(struct acpi_softc *sc PCPU_SET(switchtime, 0); PCPU_SET(switchticks, ticks); #ifdef SMP - if (wakeup_cpus != 0) - acpi_wakeup_cpus(sc, wakeup_cpus); + if (!CPU_EMPTY(&wakeup_cpus)) + acpi_wakeup_cpus(sc, &wakeup_cpus); #endif acpi_resync_clock(sc); ret = 0; @@ -291,7 +290,7 @@ acpi_sleep_machdep(struct acpi_softc *sc out: #ifdef SMP - if (wakeup_cpus != 0) + if (!CPU_EMPTY(&wakeup_cpus)) restart_cpus(wakeup_cpus); #endif Modified: projects/pseries/amd64/amd64/intr_machdep.c ============================================================================== --- projects/pseries/amd64/amd64/intr_machdep.c Thu Jun 16 19:57:07 2011 (r223160) +++ projects/pseries/amd64/amd64/intr_machdep.c Thu Jun 16 20:58:23 2011 (r223161) @@ -443,8 +443,7 @@ DB_SHOW_COMMAND(irqs, db_show_irqs) * allocate CPUs round-robin. */ -/* The BSP is always a valid target. */ -static cpumask_t intr_cpus = (1 << 0); +static cpuset_t intr_cpus; static int current_cpu; /* @@ -466,7 +465,7 @@ intr_next_cpu(void) current_cpu++; if (current_cpu > mp_maxid) current_cpu = 0; - } while (!(intr_cpus & (1 << current_cpu))); + } while (!CPU_ISSET(current_cpu, &intr_cpus)); mtx_unlock_spin(&icu_lock); return (apic_id); } @@ -497,7 +496,7 @@ intr_add_cpu(u_int cpu) printf("INTR: Adding local APIC %d as a target\n", cpu_apic_ids[cpu]); - intr_cpus |= (1 << cpu); + CPU_SET(cpu, &intr_cpus); } /* @@ -510,6 +509,9 @@ intr_shuffle_irqs(void *arg __unused) struct intsrc *isrc; int i; + /* The BSP is always a valid target. */ + CPU_SETOF(0, &intr_cpus); + /* Don't bother on UP. */ if (mp_ncpus == 1) return; Modified: projects/pseries/amd64/amd64/legacy.c ============================================================================== --- projects/pseries/amd64/amd64/legacy.c Thu Jun 16 19:57:07 2011 (r223160) +++ projects/pseries/amd64/amd64/legacy.c Thu Jun 16 20:58:23 2011 (r223161) @@ -81,6 +81,7 @@ static device_method_t legacy_methods[] DEVMETHOD(bus_read_ivar, legacy_read_ivar), DEVMETHOD(bus_write_ivar, legacy_write_ivar), DEVMETHOD(bus_alloc_resource, bus_generic_alloc_resource), + DEVMETHOD(bus_adjust_resource, bus_generic_adjust_resource), DEVMETHOD(bus_release_resource, bus_generic_release_resource), DEVMETHOD(bus_activate_resource, bus_generic_activate_resource), DEVMETHOD(bus_deactivate_resource, bus_generic_deactivate_resource), Modified: projects/pseries/amd64/amd64/machdep.c ============================================================================== --- projects/pseries/amd64/amd64/machdep.c Thu Jun 16 19:57:07 2011 (r223160) +++ projects/pseries/amd64/amd64/machdep.c Thu Jun 16 20:58:23 2011 (r223161) @@ -51,6 +51,7 @@ __FBSDID("$FreeBSD$"); #include "opt_isa.h" #include "opt_kstack_pages.h" #include "opt_maxmem.h" +#include "opt_mp_watchdog.h" #include "opt_perfmon.h" #include "opt_sched.h" #include "opt_kdtrace.h" @@ -116,6 +117,7 @@ __FBSDID("$FreeBSD$"); #include <x86/mca.h> #include <machine/md_var.h> #include <machine/metadata.h> +#include <machine/mp_watchdog.h> #include <machine/pc/bios.h> #include <machine/pcb.h> #include <machine/proc.h> @@ -734,9 +736,8 @@ cpu_idle(int busy) CTR2(KTR_SPARE2, "cpu_idle(%d) at %d", busy, curcpu); -#ifdef SMP - if (mp_grab_cpu_hlt()) - return; +#ifdef MP_WATCHDOG + ap_watchdog(PCPU_GET(cpuid)); #endif /* If we are busy - try to use fast methods. */ if (busy) { Modified: projects/pseries/amd64/amd64/mp_machdep.c ============================================================================== --- projects/pseries/amd64/amd64/mp_machdep.c Thu Jun 16 19:57:07 2011 (r223160) +++ projects/pseries/amd64/amd64/mp_machdep.c Thu Jun 16 20:58:23 2011 (r223161) @@ -29,13 +29,13 @@ __FBSDID("$FreeBSD$"); #include "opt_cpu.h" #include "opt_kstack_pages.h" -#include "opt_mp_watchdog.h" #include "opt_sched.h" #include "opt_smp.h" #include <sys/param.h> #include <sys/systm.h> #include <sys/bus.h> +#include <sys/cpuset.h> #ifdef GPROF #include <sys/gmon.h> #endif @@ -63,7 +63,6 @@ __FBSDID("$FreeBSD$"); #include <machine/cpufunc.h> #include <x86/mca.h> #include <machine/md_var.h> -#include <machine/mp_watchdog.h> #include <machine/pcb.h> #include <machine/psl.h> #include <machine/smp.h> @@ -125,7 +124,7 @@ extern inthand_t IDTVEC(fast_syscall), I * Local data and functions. */ -static volatile cpumask_t ipi_nmi_pending; +static volatile cpuset_t ipi_nmi_pending; /* used to hold the AP's until we are ready to release them */ static struct mtx ap_boot_mtx; @@ -159,11 +158,8 @@ static int start_all_aps(void); static int start_ap(int apic_id); static void release_aps(void *dummy); -static int hlt_logical_cpus; static u_int hyperthreading_cpus; /* logical cpus sharing L1 cache */ -static cpumask_t hyperthreading_cpus_mask; static int hyperthreading_allowed = 1; -static struct sysctl_ctx_list logical_cpu_clist; static u_int bootMP_size; static void @@ -241,8 +237,11 @@ topo_probe_0x4(void) * logical processors that belong to the same core * as BSP thus deducing number of threads per core. */ - cpuid_count(0x04, 0, p); - max_cores = ((p[0] >> 26) & 0x3f) + 1; + if (cpu_high >= 0x4) { + cpuid_count(0x04, 0, p); + max_cores = ((p[0] >> 26) & 0x3f) + 1; + } else + max_cores = 1; core_id_bits = mask_width(max_logical/max_cores); if (core_id_bits < 0) return; @@ -334,7 +333,7 @@ topo_probe(void) if (cpu_topo_probed) return; - logical_cpus_mask = 0; + CPU_ZERO(&logical_cpus_mask); if (mp_ncpus <= 1) cpu_cores = cpu_logical = 1; else if (cpu_vendor_id == CPU_VENDOR_AMD) @@ -478,7 +477,7 @@ cpu_mp_probe(void) * Always record BSP in CPU map so that the mbuf init code works * correctly. */ - all_cpus = 1; + CPU_SETOF(0, &all_cpus); if (mp_ncpus == 0) { /* * No CPUs were found, so this must be a UP system. Setup @@ -605,6 +604,7 @@ cpu_mp_announce(void) void init_secondary(void) { + cpuset_t tcpuset, tallcpus; struct pcpu *pc; struct nmi_pcpu *np; u_int64_t msr, cr0; @@ -736,19 +736,17 @@ init_secondary(void) CTR1(KTR_SMP, "SMP: AP CPU #%d Launched", PCPU_GET(cpuid)); printf("SMP: AP CPU #%d Launched!\n", PCPU_GET(cpuid)); + tcpuset = PCPU_GET(cpumask); /* Determine if we are a logical CPU. */ /* XXX Calculation depends on cpu_logical being a power of 2, e.g. 2 */ if (cpu_logical > 1 && PCPU_GET(apic_id) % cpu_logical != 0) - logical_cpus_mask |= PCPU_GET(cpumask); - - /* Determine if we are a hyperthread. */ - if (hyperthreading_cpus > 1 && - PCPU_GET(apic_id) % hyperthreading_cpus != 0) - hyperthreading_cpus_mask |= PCPU_GET(cpumask); + CPU_OR(&logical_cpus_mask, &tcpuset); /* Build our map of 'other' CPUs. */ - PCPU_SET(other_cpus, all_cpus & ~PCPU_GET(cpumask)); + tallcpus = all_cpus; + CPU_NAND(&tallcpus, &tcpuset); + PCPU_SET(other_cpus, tallcpus); if (bootverbose) lapic_dump("AP"); @@ -835,7 +833,7 @@ assign_cpu_ids(void) if (hyperthreading_cpus > 1 && i % hyperthreading_cpus != 0) { cpu_info[i].cpu_hyperthread = 1; -#if defined(SCHED_ULE) + /* * Don't use HT CPU if it has been disabled by a * tunable. @@ -844,7 +842,6 @@ assign_cpu_ids(void) cpu_info[i].cpu_disabled = 1; continue; } -#endif } /* Don't use this CPU if it has been disabled by a tunable. */ @@ -854,6 +851,11 @@ assign_cpu_ids(void) } } + if (hyperthreading_allowed == 0 && hyperthreading_cpus > 1) { + hyperthreading_cpus = 0; + cpu_logical = 1; + } + /* * Assign CPU IDs to local APIC IDs and disable any CPUs * beyond MAXCPU. CPU 0 is always assigned to the BSP. @@ -891,6 +893,7 @@ assign_cpu_ids(void) static int start_all_aps(void) { + cpuset_t tallcpus, tcpuset; vm_offset_t va = boot_address + KERNBASE; u_int64_t *pt4, *pt3, *pt2; u_int32_t mpbioswarmvec; @@ -955,11 +958,14 @@ start_all_aps(void) panic("AP #%d (PHY# %d) failed!", cpu, apic_id); } - all_cpus |= (1 << cpu); /* record AP in CPU map */ + CPU_SET(cpu, &all_cpus); /* record AP in CPU map */ } /* build our map of 'other' CPUs */ - PCPU_SET(other_cpus, all_cpus & ~PCPU_GET(cpumask)); + tallcpus = all_cpus; + tcpuset = PCPU_GET(cpumask); + CPU_NAND(&tallcpus, &tcpuset); + PCPU_SET(other_cpus, tallcpus); /* restore the warmstart vector */ *(u_int32_t *) WARMBOOT_OFF = mpbioswarmvec; @@ -1088,6 +1094,30 @@ SYSCTL_UINT(_debug_xhits, OID_AUTO, ipi_ #endif /* COUNT_XINVLTLB_HITS */ /* + * Send an IPI to specified CPU handling the bitmap logic. + */ +static void +ipi_send_cpu(int cpu, u_int ipi) +{ + u_int bitmap, old_pending, new_pending; + + KASSERT(cpu_apic_ids[cpu] != -1, ("IPI to non-existent CPU %d", cpu)); + + if (IPI_IS_BITMAPED(ipi)) { + bitmap = 1 << ipi; + ipi = IPI_BITMAP_VECTOR; + do { + old_pending = cpu_ipi_pending[cpu]; + new_pending = old_pending | bitmap; + } while (!atomic_cmpset_int(&cpu_ipi_pending[cpu], + old_pending, new_pending)); + if (old_pending) + return; + } + lapic_ipi_vectored(ipi, cpu_apic_ids[cpu]); +} + +/* * Flush the TLB on all other CPU's */ static void @@ -1111,28 +1141,19 @@ smp_tlb_shootdown(u_int vector, vm_offse } static void -smp_targeted_tlb_shootdown(cpumask_t mask, u_int vector, vm_offset_t addr1, vm_offset_t addr2) +smp_targeted_tlb_shootdown(cpuset_t mask, u_int vector, vm_offset_t addr1, vm_offset_t addr2) { - int ncpu, othercpus; + int cpu, ncpu, othercpus; othercpus = mp_ncpus - 1; - if (mask == (cpumask_t)-1) { - ncpu = othercpus; - if (ncpu < 1) + if (CPU_ISFULLSET(&mask)) { + if (othercpus < 1) return; } else { - mask &= ~PCPU_GET(cpumask); - if (mask == 0) - return; - ncpu = bitcount32(mask); - if (ncpu > othercpus) { - /* XXX this should be a panic offence */ - printf("SMP: tlb shootdown to %d other cpus (only have %d)\n", - ncpu, othercpus); - ncpu = othercpus; - } - /* XXX should be a panic, implied by mask == 0 above */ - if (ncpu < 1) + sched_pin(); + CPU_NAND(&mask, PCPU_PTR(cpumask)); + sched_unpin(); + if (CPU_EMPTY(&mask)) return; } if (!(read_rflags() & PSL_I)) @@ -1141,39 +1162,25 @@ smp_targeted_tlb_shootdown(cpumask_t mas smp_tlb_addr1 = addr1; smp_tlb_addr2 = addr2; atomic_store_rel_int(&smp_tlb_wait, 0); - if (mask == (cpumask_t)-1) + if (CPU_ISFULLSET(&mask)) { + ncpu = othercpus; ipi_all_but_self(vector); - else - ipi_selected(mask, vector); + } else { + ncpu = 0; + while ((cpu = cpusetobj_ffs(&mask)) != 0) { + cpu--; + CPU_CLR(cpu, &mask); + CTR3(KTR_SMP, "%s: cpu: %d ipi: %x", __func__, + cpu, vector); + ipi_send_cpu(cpu, vector); + ncpu++; + } + } while (smp_tlb_wait < ncpu) ia32_pause(); mtx_unlock_spin(&smp_ipi_mtx); } -/* - * Send an IPI to specified CPU handling the bitmap logic. - */ -static void -ipi_send_cpu(int cpu, u_int ipi) -{ - u_int bitmap, old_pending, new_pending; - - KASSERT(cpu_apic_ids[cpu] != -1, ("IPI to non-existent CPU %d", cpu)); - - if (IPI_IS_BITMAPED(ipi)) { - bitmap = 1 << ipi; - ipi = IPI_BITMAP_VECTOR; - do { - old_pending = cpu_ipi_pending[cpu]; - new_pending = old_pending | bitmap; - } while (!atomic_cmpset_int(&cpu_ipi_pending[cpu], - old_pending, new_pending)); - if (old_pending) - return; - } - lapic_ipi_vectored(ipi, cpu_apic_ids[cpu]); -} - void smp_cache_flush(void) { @@ -1220,7 +1227,7 @@ smp_invlpg_range(vm_offset_t addr1, vm_o } void -smp_masked_invltlb(cpumask_t mask) +smp_masked_invltlb(cpuset_t mask) { if (smp_started) { @@ -1232,7 +1239,7 @@ smp_masked_invltlb(cpumask_t mask) } void -smp_masked_invlpg(cpumask_t mask, vm_offset_t addr) +smp_masked_invlpg(cpuset_t mask, vm_offset_t addr) { if (smp_started) { @@ -1244,7 +1251,7 @@ smp_masked_invlpg(cpumask_t mask, vm_off } void -smp_masked_invlpg_range(cpumask_t mask, vm_offset_t addr1, vm_offset_t addr2) +smp_masked_invlpg_range(cpuset_t mask, vm_offset_t addr1, vm_offset_t addr2) { if (smp_started) { @@ -1297,7 +1304,7 @@ ipi_bitmap_handler(struct trapframe fram * send an IPI to a set of cpus. */ void -ipi_selected(cpumask_t cpus, u_int ipi) +ipi_selected(cpuset_t cpus, u_int ipi) { int cpu; @@ -1307,12 +1314,12 @@ ipi_selected(cpumask_t cpus, u_int ipi) * Set the mask of receiving CPUs for this purpose. */ if (ipi == IPI_STOP_HARD) - atomic_set_int(&ipi_nmi_pending, cpus); + CPU_OR_ATOMIC(&ipi_nmi_pending, &cpus); - CTR3(KTR_SMP, "%s: cpus: %x ipi: %x", __func__, cpus, ipi); - while ((cpu = ffs(cpus)) != 0) { + while ((cpu = cpusetobj_ffs(&cpus)) != 0) { cpu--; - cpus &= ~(1 << cpu); + CPU_CLR(cpu, &cpus); + CTR3(KTR_SMP, "%s: cpu: %d ipi: %x", __func__, cpu, ipi); ipi_send_cpu(cpu, ipi); } } @@ -1330,7 +1337,7 @@ ipi_cpu(int cpu, u_int ipi) * Set the mask of receiving CPUs for this purpose. */ if (ipi == IPI_STOP_HARD) - atomic_set_int(&ipi_nmi_pending, 1 << cpu); + CPU_SET_ATOMIC(cpu, &ipi_nmi_pending); CTR3(KTR_SMP, "%s: cpu: %d ipi: %x", __func__, cpu, ipi); ipi_send_cpu(cpu, ipi); @@ -1343,8 +1350,10 @@ void ipi_all_but_self(u_int ipi) { + sched_pin(); if (IPI_IS_BITMAPED(ipi)) { ipi_selected(PCPU_GET(other_cpus), ipi); + sched_unpin(); return; } @@ -1354,7 +1363,8 @@ ipi_all_but_self(u_int ipi) * Set the mask of receiving CPUs for this purpose. */ if (ipi == IPI_STOP_HARD) - atomic_set_int(&ipi_nmi_pending, PCPU_GET(other_cpus)); + CPU_OR_ATOMIC(&ipi_nmi_pending, PCPU_PTR(other_cpus)); + sched_unpin(); CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi); lapic_ipi_vectored(ipi, APIC_IPI_DEST_OTHERS); @@ -1363,7 +1373,7 @@ ipi_all_but_self(u_int ipi) int ipi_nmi_handler() { - cpumask_t cpumask; + cpuset_t cpumask; /* * As long as there is not a simple way to know about a NMI's @@ -1371,11 +1381,13 @@ ipi_nmi_handler() * the global pending bitword an IPI_STOP_HARD has been issued * and should be handled. */ + sched_pin(); cpumask = PCPU_GET(cpumask); - if ((ipi_nmi_pending & cpumask) == 0) + sched_unpin(); + if (!CPU_OVERLAP(&ipi_nmi_pending, &cpumask)) return (1); - atomic_clear_int(&ipi_nmi_pending, cpumask); + CPU_NAND_ATOMIC(&ipi_nmi_pending, &cpumask); cpustop_handler(); return (0); } @@ -1387,23 +1399,25 @@ ipi_nmi_handler() void cpustop_handler(void) { - cpumask_t cpumask; + cpuset_t cpumask; u_int cpu; + sched_pin(); cpu = PCPU_GET(cpuid); cpumask = PCPU_GET(cpumask); + sched_unpin(); savectx(&stoppcbs[cpu]); /* Indicate that we are stopped */ - atomic_set_int(&stopped_cpus, cpumask); + CPU_OR_ATOMIC(&stopped_cpus, &cpumask); /* Wait for restart */ - while (!(started_cpus & cpumask)) + while (!CPU_OVERLAP(&started_cpus, &cpumask)) ia32_pause(); - atomic_clear_int(&started_cpus, cpumask); - atomic_clear_int(&stopped_cpus, cpumask); + CPU_NAND_ATOMIC(&started_cpus, &cpumask); + CPU_NAND_ATOMIC(&stopped_cpus, &cpumask); if (cpu == 0 && cpustop_restartfunc != NULL) { cpustop_restartfunc(); @@ -1418,7 +1432,7 @@ cpustop_handler(void) void cpususpend_handler(void) { - cpumask_t cpumask; + cpuset_t cpumask; register_t cr3, rf; u_int cpu; @@ -1430,7 +1444,7 @@ cpususpend_handler(void) if (savectx(susppcbs[cpu])) { wbinvd(); - atomic_set_int(&stopped_cpus, cpumask); + CPU_OR_ATOMIC(&stopped_cpus, &cpumask); } else { pmap_init_pat(); PCPU_SET(switchtime, 0); @@ -1438,11 +1452,11 @@ cpususpend_handler(void) } /* Wait for resume */ - while (!(started_cpus & cpumask)) + while (!CPU_OVERLAP(&started_cpus, &cpumask)) ia32_pause(); - atomic_clear_int(&started_cpus, cpumask); - atomic_clear_int(&stopped_cpus, cpumask); + CPU_NAND_ATOMIC(&started_cpus, &cpumask); + CPU_NAND_ATOMIC(&stopped_cpus, &cpumask); /* Restore CR3 and enable interrupts */ load_cr3(cr3); @@ -1467,158 +1481,6 @@ release_aps(void *dummy __unused) } SYSINIT(start_aps, SI_SUB_SMP, SI_ORDER_FIRST, release_aps, NULL); -static int -sysctl_hlt_cpus(SYSCTL_HANDLER_ARGS) -{ - cpumask_t mask; - int error; - - mask = hlt_cpus_mask; - error = sysctl_handle_int(oidp, &mask, 0, req); - if (error || !req->newptr) - return (error); - - if (logical_cpus_mask != 0 && - (mask & logical_cpus_mask) == logical_cpus_mask) - hlt_logical_cpus = 1; - else - hlt_logical_cpus = 0; - - if (! hyperthreading_allowed) - mask |= hyperthreading_cpus_mask; - - if ((mask & all_cpus) == all_cpus) - mask &= ~(1<<0); - hlt_cpus_mask = mask; - return (error); -} -SYSCTL_PROC(_machdep, OID_AUTO, hlt_cpus, CTLTYPE_INT|CTLFLAG_RW, - 0, 0, sysctl_hlt_cpus, "IU", - "Bitmap of CPUs to halt. 101 (binary) will halt CPUs 0 and 2."); - -static int -sysctl_hlt_logical_cpus(SYSCTL_HANDLER_ARGS) -{ - int disable, error; - - disable = hlt_logical_cpus; - error = sysctl_handle_int(oidp, &disable, 0, req); - if (error || !req->newptr) - return (error); - - if (disable) - hlt_cpus_mask |= logical_cpus_mask; - else - hlt_cpus_mask &= ~logical_cpus_mask; - - if (! hyperthreading_allowed) - hlt_cpus_mask |= hyperthreading_cpus_mask; - - if ((hlt_cpus_mask & all_cpus) == all_cpus) - hlt_cpus_mask &= ~(1<<0); - - hlt_logical_cpus = disable; - return (error); -} - -static int -sysctl_hyperthreading_allowed(SYSCTL_HANDLER_ARGS) -{ - int allowed, error; - - allowed = hyperthreading_allowed; - error = sysctl_handle_int(oidp, &allowed, 0, req); - if (error || !req->newptr) - return (error); - -#ifdef SCHED_ULE - /* - * SCHED_ULE doesn't allow enabling/disabling HT cores at - * run-time. - */ - if (allowed != hyperthreading_allowed) - return (ENOTSUP); - return (error); -#endif - - if (allowed) - hlt_cpus_mask &= ~hyperthreading_cpus_mask; - else - hlt_cpus_mask |= hyperthreading_cpus_mask; - - if (logical_cpus_mask != 0 && - (hlt_cpus_mask & logical_cpus_mask) == logical_cpus_mask) - hlt_logical_cpus = 1; - else - hlt_logical_cpus = 0; - - if ((hlt_cpus_mask & all_cpus) == all_cpus) - hlt_cpus_mask &= ~(1<<0); - - hyperthreading_allowed = allowed; - return (error); -} - -static void -cpu_hlt_setup(void *dummy __unused) -{ - - if (logical_cpus_mask != 0) { - TUNABLE_INT_FETCH("machdep.hlt_logical_cpus", - &hlt_logical_cpus); - sysctl_ctx_init(&logical_cpu_clist); - SYSCTL_ADD_PROC(&logical_cpu_clist, - SYSCTL_STATIC_CHILDREN(_machdep), OID_AUTO, - "hlt_logical_cpus", CTLTYPE_INT|CTLFLAG_RW, 0, 0, - sysctl_hlt_logical_cpus, "IU", ""); - SYSCTL_ADD_UINT(&logical_cpu_clist, - SYSCTL_STATIC_CHILDREN(_machdep), OID_AUTO, - "logical_cpus_mask", CTLTYPE_INT|CTLFLAG_RD, - &logical_cpus_mask, 0, ""); - - if (hlt_logical_cpus) - hlt_cpus_mask |= logical_cpus_mask; - - /* - * If necessary for security purposes, force - * hyperthreading off, regardless of the value - * of hlt_logical_cpus. - */ - if (hyperthreading_cpus_mask) { - SYSCTL_ADD_PROC(&logical_cpu_clist, - SYSCTL_STATIC_CHILDREN(_machdep), OID_AUTO, - "hyperthreading_allowed", CTLTYPE_INT|CTLFLAG_RW, - 0, 0, sysctl_hyperthreading_allowed, "IU", ""); - if (! hyperthreading_allowed) - hlt_cpus_mask |= hyperthreading_cpus_mask; - } - } -} -SYSINIT(cpu_hlt, SI_SUB_SMP, SI_ORDER_ANY, cpu_hlt_setup, NULL); - -int -mp_grab_cpu_hlt(void) -{ - cpumask_t mask; -#ifdef MP_WATCHDOG - u_int cpuid; -#endif - int retval; - - mask = PCPU_GET(cpumask); -#ifdef MP_WATCHDOG - cpuid = PCPU_GET(cpuid); - ap_watchdog(cpuid); -#endif - - retval = 0; - while (mask & hlt_cpus_mask) { - retval = 1; - __asm __volatile("sti; hlt" : : : "memory"); - } - return (retval); -} - #ifdef COUNT_IPIS /* * Setup interrupt counters for IPI handlers. Modified: projects/pseries/amd64/amd64/pmap.c ============================================================================== --- projects/pseries/amd64/amd64/pmap.c Thu Jun 16 19:57:07 2011 (r223160) +++ projects/pseries/amd64/amd64/pmap.c Thu Jun 16 20:58:23 2011 (r223161) @@ -123,6 +123,8 @@ __FBSDID("$FreeBSD$"); #include <sys/sysctl.h> #ifdef SMP #include <sys/smp.h> +#else +#include <sys/cpuset.h> #endif #include <vm/vm.h> @@ -581,7 +583,7 @@ pmap_bootstrap(vm_paddr_t *firstaddr) PMAP_LOCK_INIT(kernel_pmap); kernel_pmap->pm_pml4 = (pdp_entry_t *)PHYS_TO_DMAP(KPML4phys); kernel_pmap->pm_root = NULL; - kernel_pmap->pm_active = -1; /* don't allow deactivation */ + CPU_FILL(&kernel_pmap->pm_active); /* don't allow deactivation */ TAILQ_INIT(&kernel_pmap->pm_pvchunk); /* @@ -923,19 +925,20 @@ pmap_update_pde_invalidate(vm_offset_t v void pmap_invalidate_page(pmap_t pmap, vm_offset_t va) { - cpumask_t cpumask, other_cpus; + cpuset_t cpumask, other_cpus; sched_pin(); - if (pmap == kernel_pmap || pmap->pm_active == all_cpus) { + if (pmap == kernel_pmap || !CPU_CMP(&pmap->pm_active, &all_cpus)) { invlpg(va); smp_invlpg(va); } else { cpumask = PCPU_GET(cpumask); other_cpus = PCPU_GET(other_cpus); - if (pmap->pm_active & cpumask) + if (CPU_OVERLAP(&pmap->pm_active, &cpumask)) invlpg(va); - if (pmap->pm_active & other_cpus) - smp_masked_invlpg(pmap->pm_active & other_cpus, va); + CPU_AND(&other_cpus, &pmap->pm_active); + if (!CPU_EMPTY(&other_cpus)) + smp_masked_invlpg(other_cpus, va); } sched_unpin(); } @@ -943,23 +946,23 @@ pmap_invalidate_page(pmap_t pmap, vm_off void pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) { - cpumask_t cpumask, other_cpus; + cpuset_t cpumask, other_cpus; vm_offset_t addr; sched_pin(); - if (pmap == kernel_pmap || pmap->pm_active == all_cpus) { + if (pmap == kernel_pmap || !CPU_CMP(&pmap->pm_active, &all_cpus)) { for (addr = sva; addr < eva; addr += PAGE_SIZE) invlpg(addr); smp_invlpg_range(sva, eva); } else { cpumask = PCPU_GET(cpumask); other_cpus = PCPU_GET(other_cpus); - if (pmap->pm_active & cpumask) + if (CPU_OVERLAP(&pmap->pm_active, &cpumask)) for (addr = sva; addr < eva; addr += PAGE_SIZE) invlpg(addr); - if (pmap->pm_active & other_cpus) - smp_masked_invlpg_range(pmap->pm_active & other_cpus, - sva, eva); + CPU_AND(&other_cpus, &pmap->pm_active); + if (!CPU_EMPTY(&other_cpus)) + smp_masked_invlpg_range(other_cpus, sva, eva); } sched_unpin(); } @@ -967,19 +970,20 @@ pmap_invalidate_range(pmap_t pmap, vm_of void pmap_invalidate_all(pmap_t pmap) { - cpumask_t cpumask, other_cpus; + cpuset_t cpumask, other_cpus; sched_pin(); - if (pmap == kernel_pmap || pmap->pm_active == all_cpus) { + if (pmap == kernel_pmap || !CPU_CMP(&pmap->pm_active, &all_cpus)) { invltlb(); smp_invltlb(); } else { cpumask = PCPU_GET(cpumask); other_cpus = PCPU_GET(other_cpus); - if (pmap->pm_active & cpumask) + if (CPU_OVERLAP(&pmap->pm_active, &cpumask)) invltlb(); - if (pmap->pm_active & other_cpus) - smp_masked_invltlb(pmap->pm_active & other_cpus); + CPU_AND(&other_cpus, &pmap->pm_active); + if (!CPU_EMPTY(&other_cpus)) + smp_masked_invltlb(other_cpus); } sched_unpin(); } @@ -995,8 +999,8 @@ pmap_invalidate_cache(void) } struct pde_action { - cpumask_t store; /* processor that updates the PDE */ - cpumask_t invalidate; /* processors that invalidate their TLB */ + cpuset_t store; /* processor that updates the PDE */ + cpuset_t invalidate; /* processors that invalidate their TLB */ vm_offset_t va; pd_entry_t *pde; pd_entry_t newpde; @@ -1007,8 +1011,12 @@ pmap_update_pde_action(void *arg) { struct pde_action *act = arg; - if (act->store == PCPU_GET(cpumask)) + sched_pin(); + if (!CPU_CMP(&act->store, PCPU_PTR(cpumask))) { + sched_unpin(); pde_store(act->pde, act->newpde); + } else + sched_unpin(); } static void @@ -1016,8 +1024,12 @@ pmap_update_pde_teardown(void *arg) { struct pde_action *act = arg; - if ((act->invalidate & PCPU_GET(cpumask)) != 0) + sched_pin(); + if (CPU_OVERLAP(&act->invalidate, PCPU_PTR(cpumask))) { + sched_unpin(); pmap_update_pde_invalidate(act->va, act->newpde); + } else + sched_unpin(); } /* @@ -1032,26 +1044,28 @@ static void pmap_update_pde(pmap_t pmap, vm_offset_t va, pd_entry_t *pde, pd_entry_t newpde) { struct pde_action act; - cpumask_t active, cpumask; + cpuset_t active, cpumask, other_cpus; sched_pin(); cpumask = PCPU_GET(cpumask); + other_cpus = PCPU_GET(other_cpus); if (pmap == kernel_pmap) active = all_cpus; else active = pmap->pm_active; - if ((active & PCPU_GET(other_cpus)) != 0) { + if (CPU_OVERLAP(&active, &other_cpus)) { act.store = cpumask; act.invalidate = active; act.va = va; act.pde = pde; act.newpde = newpde; - smp_rendezvous_cpus(cpumask | active, + CPU_OR(&cpumask, &active); + smp_rendezvous_cpus(cpumask, smp_no_rendevous_barrier, pmap_update_pde_action, pmap_update_pde_teardown, &act); } else { pde_store(pde, newpde); - if ((active & cpumask) != 0) + if (CPU_OVERLAP(&active, &cpumask)) pmap_update_pde_invalidate(va, newpde); } sched_unpin(); @@ -1065,7 +1079,7 @@ PMAP_INLINE void pmap_invalidate_page(pmap_t pmap, vm_offset_t va) { - if (pmap == kernel_pmap || pmap->pm_active) + if (pmap == kernel_pmap || !CPU_EMPTY(&pmap->pm_active)) invlpg(va); } @@ -1074,7 +1088,7 @@ pmap_invalidate_range(pmap_t pmap, vm_of { vm_offset_t addr; - if (pmap == kernel_pmap || pmap->pm_active) + if (pmap == kernel_pmap || !CPU_EMPTY(&pmap->pm_active)) for (addr = sva; addr < eva; addr += PAGE_SIZE) invlpg(addr); } @@ -1083,7 +1097,7 @@ PMAP_INLINE void pmap_invalidate_all(pmap_t pmap) { - if (pmap == kernel_pmap || pmap->pm_active) + if (pmap == kernel_pmap || !CPU_EMPTY(&pmap->pm_active)) invltlb(); } @@ -1099,7 +1113,7 @@ pmap_update_pde(pmap_t pmap, vm_offset_t { pde_store(pde, newpde); - if (pmap == kernel_pmap || pmap->pm_active) + if (pmap == kernel_pmap || !CPU_EMPTY(&pmap->pm_active)) pmap_update_pde_invalidate(va, newpde); } #endif /* !SMP */ @@ -1607,7 +1621,7 @@ pmap_pinit0(pmap_t pmap) PMAP_LOCK_INIT(pmap); pmap->pm_pml4 = (pml4_entry_t *)PHYS_TO_DMAP(KPML4phys); pmap->pm_root = NULL; - pmap->pm_active = 0; + CPU_ZERO(&pmap->pm_active); PCPU_SET(curpmap, pmap); TAILQ_INIT(&pmap->pm_pvchunk); bzero(&pmap->pm_stats, sizeof pmap->pm_stats); @@ -1649,7 +1663,7 @@ pmap_pinit(pmap_t pmap) *** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201106162058.p5GKwODG045574>