Date: Mon, 4 Jul 2011 12:04:52 +0000 (UTC) From: Attilio Rao <attilio@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r223758 - in head: contrib/top lib/libkvm share/mk sys/amd64/acpica sys/amd64/amd64 sys/cddl/dev/dtrace/amd64 sys/cddl/dev/dtrace/i386 sys/dev/xen/control sys/i386/i386 sys/i386/xen sys... Message-ID: <201107041204.p64C4q8p018157@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: attilio Date: Mon Jul 4 12:04:52 2011 New Revision: 223758 URL: http://svn.freebsd.org/changeset/base/223758 Log: With retirement of cpumask_t and usage of cpuset_t for representing a mask of CPUs, pc_other_cpus and pc_cpumask become highly inefficient. Remove them and replace their usage with custom pc_cpuid magic (as, atm, pc_cpumask can be easilly represented by (1 << pc_cpuid) and pc_other_cpus by (all_cpus & ~(1 << pc_cpuid))). This change is not targeted for MFC because of struct pcpu members removal and dependency by cpumask_t retirement. MD review by: marcel, marius, alc Tested by: pluknet MD testing by: marcel, marius, gonzo, andreast Modified: head/lib/libkvm/kvm_pcpu.c head/sys/amd64/acpica/acpi_wakeup.c head/sys/amd64/amd64/mp_machdep.c head/sys/amd64/amd64/pmap.c head/sys/amd64/amd64/vm_machdep.c head/sys/cddl/dev/dtrace/amd64/dtrace_subr.c head/sys/cddl/dev/dtrace/i386/dtrace_subr.c head/sys/dev/xen/control/control.c head/sys/i386/i386/mp_machdep.c head/sys/i386/i386/pmap.c head/sys/i386/i386/vm_machdep.c head/sys/i386/xen/mp_machdep.c head/sys/i386/xen/pmap.c head/sys/ia64/ia64/mp_machdep.c head/sys/kern/kern_rmlock.c head/sys/kern/sched_4bsd.c head/sys/kern/subr_kdb.c head/sys/kern/subr_pcpu.c head/sys/kern/subr_smp.c head/sys/mips/mips/mp_machdep.c head/sys/mips/mips/pmap.c head/sys/powerpc/aim/mmu_oea.c head/sys/powerpc/aim/mmu_oea64.c head/sys/powerpc/booke/pmap.c head/sys/powerpc/powerpc/mp_machdep.c head/sys/sparc64/sparc64/mp_machdep.c head/sys/sys/pcpu.h Directory Properties: head/ (props changed) head/cddl/contrib/opensolaris/ (props changed) head/contrib/bind9/ (props changed) head/contrib/binutils/ (props changed) head/contrib/bzip2/ (props changed) head/contrib/compiler-rt/ (props changed) head/contrib/dialog/ (props changed) head/contrib/ee/ (props changed) head/contrib/expat/ (props changed) head/contrib/file/ (props changed) head/contrib/gcc/ (props changed) head/contrib/gdb/ (props changed) head/contrib/gdtoa/ (props changed) head/contrib/gnu-sort/ (props changed) head/contrib/groff/ (props changed) head/contrib/less/ (props changed) head/contrib/libpcap/ (props changed) head/contrib/libstdc++/ (props changed) head/contrib/llvm/ (props changed) head/contrib/llvm/tools/clang/ (props changed) head/contrib/ncurses/ (props changed) head/contrib/netcat/ (props changed) head/contrib/ntp/ (props changed) head/contrib/one-true-awk/ (props changed) head/contrib/openbsm/ (props changed) head/contrib/openpam/ (props changed) head/contrib/pf/ (props changed) head/contrib/sendmail/ (props changed) head/contrib/tcpdump/ (props changed) head/contrib/tcsh/ (props changed) head/contrib/tnftp/ (props changed) head/contrib/top/ (props changed) head/contrib/top/install-sh (props changed) head/contrib/tzcode/stdtime/ (props changed) head/contrib/tzcode/zic/ (props changed) head/contrib/tzdata/ (props changed) head/contrib/wpa/ (props changed) head/contrib/xz/ (props changed) head/crypto/openssh/ (props changed) head/crypto/openssl/ (props changed) head/gnu/lib/ (props changed) head/gnu/usr.bin/binutils/ (props changed) head/gnu/usr.bin/cc/cc_tools/ (props changed) head/gnu/usr.bin/gdb/ (props changed) head/lib/libc/ (props changed) head/lib/libc/stdtime/ (props changed) head/lib/libutil/ (props changed) head/lib/libz/ (props changed) head/sbin/ (props changed) head/sbin/ipfw/ (props changed) head/share/mk/bsd.arch.inc.mk (props changed) head/share/zoneinfo/ (props changed) head/sys/ (props changed) head/sys/amd64/include/xen/ (props changed) head/sys/boot/ (props changed) head/sys/boot/i386/efi/ (props changed) head/sys/boot/ia64/efi/ (props changed) head/sys/boot/ia64/ski/ (props changed) head/sys/boot/powerpc/boot1.chrp/ (props changed) head/sys/boot/powerpc/ofw/ (props changed) head/sys/cddl/contrib/opensolaris/ (props changed) head/sys/conf/ (props changed) head/sys/contrib/dev/acpica/ (props changed) head/sys/contrib/octeon-sdk/ (props changed) head/sys/contrib/pf/ (props changed) head/sys/contrib/x86emu/ (props changed) head/usr.bin/calendar/ (props changed) head/usr.bin/csup/ (props changed) head/usr.bin/procstat/ (props changed) head/usr.sbin/ndiscvt/ (props changed) head/usr.sbin/zic/ (props changed) Modified: head/lib/libkvm/kvm_pcpu.c ============================================================================== --- head/lib/libkvm/kvm_pcpu.c Mon Jul 4 11:13:00 2011 (r223757) +++ head/lib/libkvm/kvm_pcpu.c Mon Jul 4 12:04:52 2011 (r223758) @@ -39,13 +39,11 @@ __FBSDID("$FreeBSD$"); #include <sys/param.h> -#include <sys/cpuset.h> #include <sys/pcpu.h> #include <sys/sysctl.h> #include <kvm.h> #include <limits.h> #include <stdlib.h> -#include <unistd.h> #include "kvm_private.h" @@ -120,9 +118,6 @@ _kvm_pcpu_clear(void) void * kvm_getpcpu(kvm_t *kd, int cpu) { - long kcpusetsize; - ssize_t nbytes; - uintptr_t readptr; char *buf; if (kd == NULL) { @@ -130,10 +125,6 @@ kvm_getpcpu(kvm_t *kd, int cpu) return (NULL); } - kcpusetsize = sysconf(_SC_CPUSET_SIZE); - if (kcpusetsize == -1 || (u_long)kcpusetsize > sizeof(cpuset_t)) - return ((void *)-1); - if (maxcpu == 0) if (_kvm_pcpu_init(kd) < 0) return ((void *)-1); @@ -146,26 +137,8 @@ kvm_getpcpu(kvm_t *kd, int cpu) _kvm_err(kd, kd->program, "out of memory"); return ((void *)-1); } - nbytes = sizeof(struct pcpu) - 2 * kcpusetsize; - readptr = (uintptr_t)pcpu_data[cpu]; - if (kvm_read(kd, readptr, buf, nbytes) != nbytes) { - _kvm_err(kd, kd->program, "unable to read per-CPU data"); - free(buf); - return ((void *)-1); - } - - /* Fetch the valid cpuset_t objects. */ - CPU_ZERO((cpuset_t *)(buf + nbytes)); - CPU_ZERO((cpuset_t *)(buf + nbytes + sizeof(cpuset_t))); - readptr += nbytes; - if (kvm_read(kd, readptr, buf + nbytes, kcpusetsize) != kcpusetsize) { - _kvm_err(kd, kd->program, "unable to read per-CPU data"); - free(buf); - return ((void *)-1); - } - readptr += kcpusetsize; - if (kvm_read(kd, readptr, buf + nbytes + sizeof(cpuset_t), - kcpusetsize) != kcpusetsize) { + if (kvm_read(kd, (uintptr_t)pcpu_data[cpu], buf, + sizeof(struct pcpu)) != sizeof(struct pcpu)) { _kvm_err(kd, kd->program, "unable to read per-CPU data"); free(buf); return ((void *)-1); Modified: head/sys/amd64/acpica/acpi_wakeup.c ============================================================================== --- head/sys/amd64/acpica/acpi_wakeup.c Mon Jul 4 11:13:00 2011 (r223757) +++ head/sys/amd64/acpica/acpi_wakeup.c Mon Jul 4 12:04:52 2011 (r223758) @@ -226,7 +226,8 @@ acpi_sleep_machdep(struct acpi_softc *sc return (ret); #ifdef SMP - wakeup_cpus = PCPU_GET(other_cpus); + wakeup_cpus = all_cpus; + CPU_CLR(PCPU_GET(cpuid), &wakeup_cpus); #endif AcpiSetFirmwareWakingVector(WAKECODE_PADDR(sc)); Modified: head/sys/amd64/amd64/mp_machdep.c ============================================================================== --- head/sys/amd64/amd64/mp_machdep.c Mon Jul 4 11:13:00 2011 (r223757) +++ head/sys/amd64/amd64/mp_machdep.c Mon Jul 4 12:04:52 2011 (r223758) @@ -604,10 +604,10 @@ cpu_mp_announce(void) void init_secondary(void) { - cpuset_t tcpuset, tallcpus; struct pcpu *pc; struct nmi_pcpu *np; u_int64_t msr, cr0; + u_int cpuid; int cpu, gsel_tss, x; struct region_descriptor ap_gdt; @@ -711,8 +711,9 @@ init_secondary(void) fpuinit(); /* A quick check from sanity claus */ + cpuid = PCPU_GET(cpuid); if (PCPU_GET(apic_id) != lapic_id()) { - printf("SMP: cpuid = %d\n", PCPU_GET(cpuid)); + printf("SMP: cpuid = %d\n", cpuid); printf("SMP: actual apic_id = %d\n", lapic_id()); printf("SMP: correct apic_id = %d\n", PCPU_GET(apic_id)); panic("cpuid mismatch! boom!!"); @@ -734,19 +735,13 @@ init_secondary(void) smp_cpus++; - CTR1(KTR_SMP, "SMP: AP CPU #%d Launched", PCPU_GET(cpuid)); - printf("SMP: AP CPU #%d Launched!\n", PCPU_GET(cpuid)); - tcpuset = PCPU_GET(cpumask); + CTR1(KTR_SMP, "SMP: AP CPU #%d Launched", cpuid); + printf("SMP: AP CPU #%d Launched!\n", cpuid); /* Determine if we are a logical CPU. */ /* XXX Calculation depends on cpu_logical being a power of 2, e.g. 2 */ if (cpu_logical > 1 && PCPU_GET(apic_id) % cpu_logical != 0) - CPU_OR(&logical_cpus_mask, &tcpuset); - - /* Build our map of 'other' CPUs. */ - tallcpus = all_cpus; - CPU_NAND(&tallcpus, &tcpuset); - PCPU_SET(other_cpus, tallcpus); + CPU_SET(cpuid, &logical_cpus_mask); if (bootverbose) lapic_dump("AP"); @@ -893,7 +888,6 @@ assign_cpu_ids(void) static int start_all_aps(void) { - cpuset_t tallcpus, tcpuset; vm_offset_t va = boot_address + KERNBASE; u_int64_t *pt4, *pt3, *pt2; u_int32_t mpbioswarmvec; @@ -961,12 +955,6 @@ start_all_aps(void) CPU_SET(cpu, &all_cpus); /* record AP in CPU map */ } - /* build our map of 'other' CPUs */ - tallcpus = all_cpus; - tcpuset = PCPU_GET(cpumask); - CPU_NAND(&tallcpus, &tcpuset); - PCPU_SET(other_cpus, tallcpus); - /* restore the warmstart vector */ *(u_int32_t *) WARMBOOT_OFF = mpbioswarmvec; @@ -1150,9 +1138,7 @@ smp_targeted_tlb_shootdown(cpuset_t mask if (othercpus < 1) return; } else { - sched_pin(); - CPU_NAND(&mask, PCPU_PTR(cpumask)); - sched_unpin(); + CPU_CLR(PCPU_GET(cpuid), &mask); if (CPU_EMPTY(&mask)) return; } @@ -1349,11 +1335,13 @@ ipi_cpu(int cpu, u_int ipi) void ipi_all_but_self(u_int ipi) { + cpuset_t other_cpus; + + other_cpus = all_cpus; + CPU_CLR(PCPU_GET(cpuid), &other_cpus); - sched_pin(); if (IPI_IS_BITMAPED(ipi)) { - ipi_selected(PCPU_GET(other_cpus), ipi); - sched_unpin(); + ipi_selected(other_cpus, ipi); return; } @@ -1363,8 +1351,7 @@ ipi_all_but_self(u_int ipi) * Set the mask of receiving CPUs for this purpose. */ if (ipi == IPI_STOP_HARD) - CPU_OR_ATOMIC(&ipi_nmi_pending, PCPU_PTR(other_cpus)); - sched_unpin(); + CPU_OR_ATOMIC(&ipi_nmi_pending, &other_cpus); CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi); lapic_ipi_vectored(ipi, APIC_IPI_DEST_OTHERS); @@ -1373,7 +1360,7 @@ ipi_all_but_self(u_int ipi) int ipi_nmi_handler() { - cpuset_t cpumask; + u_int cpuid; /* * As long as there is not a simple way to know about a NMI's @@ -1381,13 +1368,11 @@ ipi_nmi_handler() * the global pending bitword an IPI_STOP_HARD has been issued * and should be handled. */ - sched_pin(); - cpumask = PCPU_GET(cpumask); - sched_unpin(); - if (!CPU_OVERLAP(&ipi_nmi_pending, &cpumask)) + cpuid = PCPU_GET(cpuid); + if (!CPU_ISSET(cpuid, &ipi_nmi_pending)) return (1); - CPU_NAND_ATOMIC(&ipi_nmi_pending, &cpumask); + CPU_CLR_ATOMIC(cpuid, &ipi_nmi_pending); cpustop_handler(); return (0); } @@ -1399,25 +1384,21 @@ ipi_nmi_handler() void cpustop_handler(void) { - cpuset_t cpumask; u_int cpu; - sched_pin(); cpu = PCPU_GET(cpuid); - cpumask = PCPU_GET(cpumask); - sched_unpin(); savectx(&stoppcbs[cpu]); /* Indicate that we are stopped */ - CPU_OR_ATOMIC(&stopped_cpus, &cpumask); + CPU_SET_ATOMIC(cpu, &stopped_cpus); /* Wait for restart */ - while (!CPU_OVERLAP(&started_cpus, &cpumask)) + while (!CPU_ISSET(cpu, &started_cpus)) ia32_pause(); - CPU_NAND_ATOMIC(&started_cpus, &cpumask); - CPU_NAND_ATOMIC(&stopped_cpus, &cpumask); + CPU_CLR_ATOMIC(cpu, &started_cpus); + CPU_CLR_ATOMIC(cpu, &stopped_cpus); if (cpu == 0 && cpustop_restartfunc != NULL) { cpustop_restartfunc(); @@ -1432,19 +1413,17 @@ cpustop_handler(void) void cpususpend_handler(void) { - cpuset_t cpumask; register_t cr3, rf; u_int cpu; cpu = PCPU_GET(cpuid); - cpumask = PCPU_GET(cpumask); rf = intr_disable(); cr3 = rcr3(); if (savectx(susppcbs[cpu])) { wbinvd(); - CPU_OR_ATOMIC(&stopped_cpus, &cpumask); + CPU_SET_ATOMIC(cpu, &stopped_cpus); } else { pmap_init_pat(); PCPU_SET(switchtime, 0); @@ -1452,11 +1431,11 @@ cpususpend_handler(void) } /* Wait for resume */ - while (!CPU_OVERLAP(&started_cpus, &cpumask)) + while (!CPU_ISSET(cpu, &started_cpus)) ia32_pause(); - CPU_NAND_ATOMIC(&started_cpus, &cpumask); - CPU_NAND_ATOMIC(&stopped_cpus, &cpumask); + CPU_CLR_ATOMIC(cpu, &started_cpus); + CPU_CLR_ATOMIC(cpu, &stopped_cpus); /* Restore CR3 and enable interrupts */ load_cr3(cr3); Modified: head/sys/amd64/amd64/pmap.c ============================================================================== --- head/sys/amd64/amd64/pmap.c Mon Jul 4 11:13:00 2011 (r223757) +++ head/sys/amd64/amd64/pmap.c Mon Jul 4 12:04:52 2011 (r223758) @@ -925,16 +925,18 @@ pmap_update_pde_invalidate(vm_offset_t v void pmap_invalidate_page(pmap_t pmap, vm_offset_t va) { - cpuset_t cpumask, other_cpus; + cpuset_t other_cpus; + u_int cpuid; sched_pin(); if (pmap == kernel_pmap || !CPU_CMP(&pmap->pm_active, &all_cpus)) { invlpg(va); smp_invlpg(va); } else { - cpumask = PCPU_GET(cpumask); - other_cpus = PCPU_GET(other_cpus); - if (CPU_OVERLAP(&pmap->pm_active, &cpumask)) + cpuid = PCPU_GET(cpuid); + other_cpus = all_cpus; + CPU_CLR(cpuid, &other_cpus); + if (CPU_ISSET(cpuid, &pmap->pm_active)) invlpg(va); CPU_AND(&other_cpus, &pmap->pm_active); if (!CPU_EMPTY(&other_cpus)) @@ -946,8 +948,9 @@ pmap_invalidate_page(pmap_t pmap, vm_off void pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) { - cpuset_t cpumask, other_cpus; + cpuset_t other_cpus; vm_offset_t addr; + u_int cpuid; sched_pin(); if (pmap == kernel_pmap || !CPU_CMP(&pmap->pm_active, &all_cpus)) { @@ -955,9 +958,10 @@ pmap_invalidate_range(pmap_t pmap, vm_of invlpg(addr); smp_invlpg_range(sva, eva); } else { - cpumask = PCPU_GET(cpumask); - other_cpus = PCPU_GET(other_cpus); - if (CPU_OVERLAP(&pmap->pm_active, &cpumask)) + cpuid = PCPU_GET(cpuid); + other_cpus = all_cpus; + CPU_CLR(cpuid, &other_cpus); + if (CPU_ISSET(cpuid, &pmap->pm_active)) for (addr = sva; addr < eva; addr += PAGE_SIZE) invlpg(addr); CPU_AND(&other_cpus, &pmap->pm_active); @@ -970,16 +974,18 @@ pmap_invalidate_range(pmap_t pmap, vm_of void pmap_invalidate_all(pmap_t pmap) { - cpuset_t cpumask, other_cpus; + cpuset_t other_cpus; + u_int cpuid; sched_pin(); if (pmap == kernel_pmap || !CPU_CMP(&pmap->pm_active, &all_cpus)) { invltlb(); smp_invltlb(); } else { - cpumask = PCPU_GET(cpumask); - other_cpus = PCPU_GET(other_cpus); - if (CPU_OVERLAP(&pmap->pm_active, &cpumask)) + cpuid = PCPU_GET(cpuid); + other_cpus = all_cpus; + CPU_CLR(cpuid, &other_cpus); + if (CPU_ISSET(cpuid, &pmap->pm_active)) invltlb(); CPU_AND(&other_cpus, &pmap->pm_active); if (!CPU_EMPTY(&other_cpus)) @@ -999,11 +1005,11 @@ pmap_invalidate_cache(void) } struct pde_action { - cpuset_t store; /* processor that updates the PDE */ cpuset_t invalidate; /* processors that invalidate their TLB */ vm_offset_t va; pd_entry_t *pde; pd_entry_t newpde; + u_int store; /* processor that updates the PDE */ }; static void @@ -1011,12 +1017,8 @@ pmap_update_pde_action(void *arg) { struct pde_action *act = arg; - sched_pin(); - if (!CPU_CMP(&act->store, PCPU_PTR(cpumask))) { - sched_unpin(); + if (act->store == PCPU_GET(cpuid)) pde_store(act->pde, act->newpde); - } else - sched_unpin(); } static void @@ -1024,12 +1026,8 @@ pmap_update_pde_teardown(void *arg) { struct pde_action *act = arg; - sched_pin(); - if (CPU_OVERLAP(&act->invalidate, PCPU_PTR(cpumask))) { - sched_unpin(); + if (CPU_ISSET(PCPU_GET(cpuid), &act->invalidate)) pmap_update_pde_invalidate(act->va, act->newpde); - } else - sched_unpin(); } /* @@ -1044,28 +1042,30 @@ static void pmap_update_pde(pmap_t pmap, vm_offset_t va, pd_entry_t *pde, pd_entry_t newpde) { struct pde_action act; - cpuset_t active, cpumask, other_cpus; + cpuset_t active, other_cpus; + u_int cpuid; sched_pin(); - cpumask = PCPU_GET(cpumask); - other_cpus = PCPU_GET(other_cpus); + cpuid = PCPU_GET(cpuid); + other_cpus = all_cpus; + CPU_CLR(cpuid, &other_cpus); if (pmap == kernel_pmap) active = all_cpus; else active = pmap->pm_active; if (CPU_OVERLAP(&active, &other_cpus)) { - act.store = cpumask; + act.store = cpuid; act.invalidate = active; act.va = va; act.pde = pde; act.newpde = newpde; - CPU_OR(&cpumask, &active); - smp_rendezvous_cpus(cpumask, + CPU_SET(cpuid, &active); + smp_rendezvous_cpus(active, smp_no_rendevous_barrier, pmap_update_pde_action, pmap_update_pde_teardown, &act); } else { pde_store(pde, newpde); - if (CPU_OVERLAP(&active, &cpumask)) + if (CPU_ISSET(cpuid, &active)) pmap_update_pde_invalidate(va, newpde); } sched_unpin(); @@ -5095,17 +5095,19 @@ void pmap_activate(struct thread *td) { pmap_t pmap, oldpmap; + u_int cpuid; u_int64_t cr3; critical_enter(); pmap = vmspace_pmap(td->td_proc->p_vmspace); oldpmap = PCPU_GET(curpmap); + cpuid = PCPU_GET(cpuid); #ifdef SMP - CPU_NAND_ATOMIC(&oldpmap->pm_active, PCPU_PTR(cpumask)); - CPU_OR_ATOMIC(&pmap->pm_active, PCPU_PTR(cpumask)); + CPU_CLR_ATOMIC(cpuid, &oldpmap->pm_active); + CPU_SET_ATOMIC(cpuid, &pmap->pm_active); #else - CPU_NAND(&oldpmap->pm_active, PCPU_PTR(cpumask)); - CPU_OR(&pmap->pm_active, PCPU_PTR(cpumask)); + CPU_CLR(cpuid, &oldpmap->pm_active); + CPU_SET(cpuid, &pmap->pm_active); #endif cr3 = DMAP_TO_PHYS((vm_offset_t)pmap->pm_pml4); td->td_pcb->pcb_cr3 = cr3; Modified: head/sys/amd64/amd64/vm_machdep.c ============================================================================== --- head/sys/amd64/amd64/vm_machdep.c Mon Jul 4 11:13:00 2011 (r223757) +++ head/sys/amd64/amd64/vm_machdep.c Mon Jul 4 12:04:52 2011 (r223758) @@ -59,7 +59,6 @@ __FBSDID("$FreeBSD$"); #include <sys/mutex.h> #include <sys/pioctl.h> #include <sys/proc.h> -#include <sys/sched.h> #include <sys/sf_buf.h> #include <sys/smp.h> #include <sys/sysctl.h> @@ -535,8 +534,8 @@ cpu_reset() u_int cnt; if (smp_active) { - sched_pin(); - map = PCPU_GET(other_cpus); + map = all_cpus; + CPU_CLR(PCPU_GET(cpuid), &map); CPU_NAND(&map, &stopped_cpus); if (!CPU_EMPTY(&map)) { printf("cpu_reset: Stopping other CPUs\n"); @@ -545,7 +544,6 @@ cpu_reset() if (PCPU_GET(cpuid) != 0) { cpu_reset_proxyid = PCPU_GET(cpuid); - sched_unpin(); cpustop_restartfunc = cpu_reset_proxy; cpu_reset_proxy_active = 0; printf("cpu_reset: Restarting BSP\n"); @@ -564,8 +562,7 @@ cpu_reset() while (1); /* NOTREACHED */ - } else - sched_unpin(); + } DELAY(1000000); } Modified: head/sys/cddl/dev/dtrace/amd64/dtrace_subr.c ============================================================================== --- head/sys/cddl/dev/dtrace/amd64/dtrace_subr.c Mon Jul 4 11:13:00 2011 (r223757) +++ head/sys/cddl/dev/dtrace/amd64/dtrace_subr.c Mon Jul 4 12:04:52 2011 (r223758) @@ -412,8 +412,8 @@ dtrace_gethrtime_init(void *arg) continue; pc = pcpu_find(i); - map = PCPU_GET(cpumask); - CPU_OR(&map, &pc->pc_cpumask); + CPU_SETOF(PCPU_GET(cpuid), &map); + CPU_SET(pc->pc_cpuid, &map); smp_rendezvous_cpus(map, NULL, dtrace_gethrtime_init_cpu, Modified: head/sys/cddl/dev/dtrace/i386/dtrace_subr.c ============================================================================== --- head/sys/cddl/dev/dtrace/i386/dtrace_subr.c Mon Jul 4 11:13:00 2011 (r223757) +++ head/sys/cddl/dev/dtrace/i386/dtrace_subr.c Mon Jul 4 12:04:52 2011 (r223758) @@ -413,8 +413,8 @@ dtrace_gethrtime_init(void *arg) continue; pc = pcpu_find(i); - map = PCPU_GET(cpumask); - CPU_OR(&map, &pc->pc_cpumask); + CPU_SETOF(PCPU_GET(cpuid), &map); + CPU_SET(pc->pc_cpuid, &map); smp_rendezvous_cpus(map, NULL, dtrace_gethrtime_init_cpu, Modified: head/sys/dev/xen/control/control.c ============================================================================== --- head/sys/dev/xen/control/control.c Mon Jul 4 11:13:00 2011 (r223757) +++ head/sys/dev/xen/control/control.c Mon Jul 4 12:04:52 2011 (r223758) @@ -197,6 +197,7 @@ extern void xencons_resume(void); static void xctrl_suspend() { + u_int cpuid; int i, j, k, fpp; unsigned long max_pfn, start_info_mfn; @@ -210,11 +211,11 @@ xctrl_suspend() thread_lock(td); sched_bind(td, 0); thread_unlock(td); - KASSERT(PCPU_GET(cpuid) == 0, ("xen_suspend: not running on cpu 0")); + cpuid = PCPU_GET(cpuid); + KASSERT(cpuid == 0, ("xen_suspend: not running on cpu 0")); - sched_pin(); - map = PCPU_GET(other_cpus); - sched_unpin(); + map = all_cpus; + CPU_CLR(cpuid, &map); CPU_NAND(&map, &stopped_cpus); if (!CPU_EMPTY(&map)) stop_cpus(map); Modified: head/sys/i386/i386/mp_machdep.c ============================================================================== --- head/sys/i386/i386/mp_machdep.c Mon Jul 4 11:13:00 2011 (r223757) +++ head/sys/i386/i386/mp_machdep.c Mon Jul 4 12:04:52 2011 (r223758) @@ -658,12 +658,11 @@ cpu_mp_announce(void) void init_secondary(void) { - cpuset_t tcpuset, tallcpus; struct pcpu *pc; vm_offset_t addr; int gsel_tss; int x, myid; - u_int cr0; + u_int cpuid, cr0; /* bootAP is set in start_ap() to our ID. */ myid = bootAP; @@ -758,8 +757,9 @@ init_secondary(void) #endif /* A quick check from sanity claus */ + cpuid = PCPU_GET(cpuid); if (PCPU_GET(apic_id) != lapic_id()) { - printf("SMP: cpuid = %d\n", PCPU_GET(cpuid)); + printf("SMP: cpuid = %d\n", cpuid); printf("SMP: actual apic_id = %d\n", lapic_id()); printf("SMP: correct apic_id = %d\n", PCPU_GET(apic_id)); panic("cpuid mismatch! boom!!"); @@ -781,19 +781,13 @@ init_secondary(void) smp_cpus++; - CTR1(KTR_SMP, "SMP: AP CPU #%d Launched", PCPU_GET(cpuid)); - printf("SMP: AP CPU #%d Launched!\n", PCPU_GET(cpuid)); - tcpuset = PCPU_GET(cpumask); + CTR1(KTR_SMP, "SMP: AP CPU #%d Launched", cpuid); + printf("SMP: AP CPU #%d Launched!\n", cpuid); /* Determine if we are a logical CPU. */ /* XXX Calculation depends on cpu_logical being a power of 2, e.g. 2 */ if (cpu_logical > 1 && PCPU_GET(apic_id) % cpu_logical != 0) - CPU_OR(&logical_cpus_mask, &tcpuset); - - /* Build our map of 'other' CPUs. */ - tallcpus = all_cpus; - CPU_NAND(&tallcpus, &tcpuset); - PCPU_SET(other_cpus, tallcpus); + CPU_SET(cpuid, &logical_cpus_mask); if (bootverbose) lapic_dump("AP"); @@ -934,7 +928,6 @@ assign_cpu_ids(void) static int start_all_aps(void) { - cpuset_t tallcpus; #ifndef PC98 u_char mpbiosreason; #endif @@ -997,11 +990,6 @@ start_all_aps(void) CPU_SET(cpu, &all_cpus); /* record AP in CPU map */ } - /* build our map of 'other' CPUs */ - tallcpus = all_cpus; - CPU_NAND(&tallcpus, PCPU_PTR(cpumask)); - PCPU_SET(other_cpus, tallcpus); - /* restore the warmstart vector */ *(u_int32_t *) WARMBOOT_OFF = mpbioswarmvec; @@ -1253,9 +1241,7 @@ smp_targeted_tlb_shootdown(cpuset_t mask if (othercpus < 1) return; } else { - sched_pin(); - CPU_NAND(&mask, PCPU_PTR(cpumask)); - sched_unpin(); + CPU_CLR(PCPU_GET(cpuid), &mask); if (CPU_EMPTY(&mask)) return; } @@ -1452,11 +1438,12 @@ ipi_cpu(int cpu, u_int ipi) void ipi_all_but_self(u_int ipi) { + cpuset_t other_cpus; - sched_pin(); + other_cpus = all_cpus; + CPU_CLR(PCPU_GET(cpuid), &other_cpus); if (IPI_IS_BITMAPED(ipi)) { - ipi_selected(PCPU_GET(other_cpus), ipi); - sched_unpin(); + ipi_selected(other_cpus, ipi); return; } @@ -1466,8 +1453,7 @@ ipi_all_but_self(u_int ipi) * Set the mask of receiving CPUs for this purpose. */ if (ipi == IPI_STOP_HARD) - CPU_OR_ATOMIC(&ipi_nmi_pending, PCPU_PTR(other_cpus)); - sched_unpin(); + CPU_OR_ATOMIC(&ipi_nmi_pending, &other_cpus); CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi); lapic_ipi_vectored(ipi, APIC_IPI_DEST_OTHERS); @@ -1476,7 +1462,7 @@ ipi_all_but_self(u_int ipi) int ipi_nmi_handler() { - cpuset_t cpumask; + u_int cpuid; /* * As long as there is not a simple way to know about a NMI's @@ -1484,13 +1470,11 @@ ipi_nmi_handler() * the global pending bitword an IPI_STOP_HARD has been issued * and should be handled. */ - sched_pin(); - cpumask = PCPU_GET(cpumask); - sched_unpin(); - if (!CPU_OVERLAP(&ipi_nmi_pending, &cpumask)) + cpuid = PCPU_GET(cpuid); + if (!CPU_ISSET(cpuid, &ipi_nmi_pending)) return (1); - CPU_NAND_ATOMIC(&ipi_nmi_pending, &cpumask); + CPU_CLR_ATOMIC(cpuid, &ipi_nmi_pending); cpustop_handler(); return (0); } @@ -1502,25 +1486,21 @@ ipi_nmi_handler() void cpustop_handler(void) { - cpuset_t cpumask; u_int cpu; - sched_pin(); cpu = PCPU_GET(cpuid); - cpumask = PCPU_GET(cpumask); - sched_unpin(); savectx(&stoppcbs[cpu]); /* Indicate that we are stopped */ - CPU_OR_ATOMIC(&stopped_cpus, &cpumask); + CPU_SET_ATOMIC(cpu, &stopped_cpus); /* Wait for restart */ - while (!CPU_OVERLAP(&started_cpus, &cpumask)) + while (!CPU_ISSET(cpu, &started_cpus)) ia32_pause(); - CPU_NAND_ATOMIC(&started_cpus, &cpumask); - CPU_NAND_ATOMIC(&stopped_cpus, &cpumask); + CPU_CLR_ATOMIC(cpu, &started_cpus); + CPU_CLR_ATOMIC(cpu, &stopped_cpus); if (cpu == 0 && cpustop_restartfunc != NULL) { cpustop_restartfunc(); Modified: head/sys/i386/i386/pmap.c ============================================================================== --- head/sys/i386/i386/pmap.c Mon Jul 4 11:13:00 2011 (r223757) +++ head/sys/i386/i386/pmap.c Mon Jul 4 12:04:52 2011 (r223758) @@ -932,16 +932,18 @@ pmap_update_pde_invalidate(vm_offset_t v void pmap_invalidate_page(pmap_t pmap, vm_offset_t va) { - cpuset_t cpumask, other_cpus; + cpuset_t other_cpus; + u_int cpuid; sched_pin(); if (pmap == kernel_pmap || !CPU_CMP(&pmap->pm_active, &all_cpus)) { invlpg(va); smp_invlpg(va); } else { - cpumask = PCPU_GET(cpumask); - other_cpus = PCPU_GET(other_cpus); - if (CPU_OVERLAP(&pmap->pm_active, &cpumask)) + cpuid = PCPU_GET(cpuid); + other_cpus = all_cpus; + CPU_CLR(cpuid, &other_cpus); + if (CPU_ISSET(cpuid, &pmap->pm_active)) invlpg(va); CPU_AND(&other_cpus, &pmap->pm_active); if (!CPU_EMPTY(&other_cpus)) @@ -953,8 +955,9 @@ pmap_invalidate_page(pmap_t pmap, vm_off void pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) { - cpuset_t cpumask, other_cpus; + cpuset_t other_cpus; vm_offset_t addr; + u_int cpuid; sched_pin(); if (pmap == kernel_pmap || !CPU_CMP(&pmap->pm_active, &all_cpus)) { @@ -962,9 +965,10 @@ pmap_invalidate_range(pmap_t pmap, vm_of invlpg(addr); smp_invlpg_range(sva, eva); } else { - cpumask = PCPU_GET(cpumask); - other_cpus = PCPU_GET(other_cpus); - if (CPU_OVERLAP(&pmap->pm_active, &cpumask)) + cpuid = PCPU_GET(cpuid); + other_cpus = all_cpus; + CPU_CLR(cpuid, &other_cpus); + if (CPU_ISSET(cpuid, &pmap->pm_active)) for (addr = sva; addr < eva; addr += PAGE_SIZE) invlpg(addr); CPU_AND(&other_cpus, &pmap->pm_active); @@ -977,16 +981,18 @@ pmap_invalidate_range(pmap_t pmap, vm_of void pmap_invalidate_all(pmap_t pmap) { - cpuset_t cpumask, other_cpus; + cpuset_t other_cpus; + u_int cpuid; sched_pin(); if (pmap == kernel_pmap || !CPU_CMP(&pmap->pm_active, &all_cpus)) { invltlb(); smp_invltlb(); } else { - cpumask = PCPU_GET(cpumask); - other_cpus = PCPU_GET(other_cpus); - if (CPU_OVERLAP(&pmap->pm_active, &cpumask)) + cpuid = PCPU_GET(cpuid); + other_cpus = all_cpus; + CPU_CLR(cpuid, &other_cpus); + if (CPU_ISSET(cpuid, &pmap->pm_active)) invltlb(); CPU_AND(&other_cpus, &pmap->pm_active); if (!CPU_EMPTY(&other_cpus)) @@ -1006,11 +1012,11 @@ pmap_invalidate_cache(void) } struct pde_action { - cpuset_t store; /* processor that updates the PDE */ cpuset_t invalidate; /* processors that invalidate their TLB */ vm_offset_t va; pd_entry_t *pde; pd_entry_t newpde; + u_int store; /* processor that updates the PDE */ }; static void @@ -1020,9 +1026,7 @@ pmap_update_pde_kernel(void *arg) pd_entry_t *pde; pmap_t pmap; - sched_pin(); - if (!CPU_CMP(&act->store, PCPU_PTR(cpumask))) { - sched_unpin(); + if (act->store == PCPU_GET(cpuid)) { /* * Elsewhere, this operation requires allpmaps_lock for @@ -1033,8 +1037,7 @@ pmap_update_pde_kernel(void *arg) pde = pmap_pde(pmap, act->va); pde_store(pde, act->newpde); } - } else - sched_unpin(); + } } static void @@ -1042,12 +1045,8 @@ pmap_update_pde_user(void *arg) { struct pde_action *act = arg; - sched_pin(); - if (!CPU_CMP(&act->store, PCPU_PTR(cpumask))) { - sched_unpin(); + if (act->store == PCPU_GET(cpuid)) pde_store(act->pde, act->newpde); - } else - sched_unpin(); } static void @@ -1055,12 +1054,8 @@ pmap_update_pde_teardown(void *arg) { struct pde_action *act = arg; - sched_pin(); - if (CPU_OVERLAP(&act->invalidate, PCPU_PTR(cpumask))) { - sched_unpin(); + if (CPU_ISSET(PCPU_GET(cpuid), &act->invalidate)) pmap_update_pde_invalidate(act->va, act->newpde); - } else - sched_unpin(); } /* @@ -1075,23 +1070,25 @@ static void pmap_update_pde(pmap_t pmap, vm_offset_t va, pd_entry_t *pde, pd_entry_t newpde) { struct pde_action act; - cpuset_t active, cpumask, other_cpus; + cpuset_t active, other_cpus; + u_int cpuid; sched_pin(); - cpumask = PCPU_GET(cpumask); - other_cpus = PCPU_GET(other_cpus); + cpuid = PCPU_GET(cpuid); + other_cpus = all_cpus; + CPU_CLR(cpuid, &other_cpus); if (pmap == kernel_pmap) active = all_cpus; else active = pmap->pm_active; if (CPU_OVERLAP(&active, &other_cpus)) { - act.store = cpumask; + act.store = cpuid; act.invalidate = active; act.va = va; act.pde = pde; act.newpde = newpde; - CPU_OR(&cpumask, &active); - smp_rendezvous_cpus(cpumask, + CPU_SET(cpuid, &active); + smp_rendezvous_cpus(active, smp_no_rendevous_barrier, pmap == kernel_pmap ? pmap_update_pde_kernel : pmap_update_pde_user, pmap_update_pde_teardown, &act); @@ -1100,7 +1097,7 @@ pmap_update_pde(pmap_t pmap, vm_offset_t pmap_kenter_pde(va, newpde); else pde_store(pde, newpde); - if (CPU_OVERLAP(&active, &cpumask)) + if (CPU_ISSET(cpuid, &active)) pmap_update_pde_invalidate(va, newpde); } sched_unpin(); @@ -1925,12 +1922,12 @@ pmap_lazyfix_action(void) } static void -pmap_lazyfix_self(cpuset_t mymask) +pmap_lazyfix_self(u_int cpuid) { if (rcr3() == lazyptd) load_cr3(PCPU_GET(curpcb)->pcb_cr3); - CPU_NAND_ATOMIC(lazymask, &mymask); + CPU_CLR_ATOMIC(cpuid, lazymask); } @@ -1938,7 +1935,7 @@ static void pmap_lazyfix(pmap_t pmap) { cpuset_t mymask, mask; - u_int spins; + u_int cpuid, spins; int lsb; mask = pmap->pm_active; @@ -1956,10 +1953,13 @@ pmap_lazyfix(pmap_t pmap) #else lazyptd = vtophys(pmap->pm_pdir); #endif - mymask = PCPU_GET(cpumask); + cpuid = PCPU_GET(cpuid); + + /* Use a cpuset just for having an easy check. */ + CPU_SETOF(cpuid, &mymask); if (!CPU_CMP(&mask, &mymask)) { lazymask = &pmap->pm_active; - pmap_lazyfix_self(mymask); + pmap_lazyfix_self(cpuid); } else { atomic_store_rel_int((u_int *)&lazymask, (u_int)&pmap->pm_active); @@ -5098,17 +5098,19 @@ void pmap_activate(struct thread *td) { pmap_t pmap, oldpmap; + u_int cpuid; u_int32_t cr3; critical_enter(); pmap = vmspace_pmap(td->td_proc->p_vmspace); oldpmap = PCPU_GET(curpmap); + cpuid = PCPU_GET(cpuid); #if defined(SMP) - CPU_NAND_ATOMIC(&oldpmap->pm_active, PCPU_PTR(cpumask)); - CPU_OR_ATOMIC(&pmap->pm_active, PCPU_PTR(cpumask)); + CPU_CLR_ATOMIC(cpuid, &oldpmap->pm_active); + CPU_SET_ATOMIC(cpuid, &pmap->pm_active); #else - CPU_NAND(&oldpmap->pm_active, PCPU_PTR(cpumask)); - CPU_OR(&pmap->pm_active, PCPU_PTR(cpumask)); + CPU_CLR(cpuid, &oldpmap->pm_active); + CPU_SET(cpuid, &pmap->pm_active); #endif #ifdef PAE cr3 = vtophys(pmap->pm_pdpt); Modified: head/sys/i386/i386/vm_machdep.c ============================================================================== --- head/sys/i386/i386/vm_machdep.c Mon Jul 4 11:13:00 2011 (r223757) +++ head/sys/i386/i386/vm_machdep.c Mon Jul 4 12:04:52 2011 (r223758) @@ -602,8 +602,8 @@ cpu_reset() u_int cnt; if (smp_active) { - sched_pin(); - map = PCPU_GET(other_cpus); + map = all_cpus; + CPU_CLR(PCPU_GET(cpuid), &map); CPU_NAND(&map, &stopped_cpus); if (!CPU_EMPTY(&map)) { printf("cpu_reset: Stopping other CPUs\n"); @@ -612,7 +612,6 @@ cpu_reset() if (PCPU_GET(cpuid) != 0) { cpu_reset_proxyid = PCPU_GET(cpuid); - sched_unpin(); cpustop_restartfunc = cpu_reset_proxy; cpu_reset_proxy_active = 0; printf("cpu_reset: Restarting BSP\n"); @@ -632,8 +631,7 @@ cpu_reset() while (1); /* NOTREACHED */ - } else - sched_unpin(); + } DELAY(1000000); } @@ -802,7 +800,8 @@ sf_buf_alloc(struct vm_page *m, int flag struct sf_head *hash_list; struct sf_buf *sf; #ifdef SMP - cpuset_t cpumask, other_cpus; + cpuset_t other_cpus; + u_int cpuid; #endif int error; @@ -877,13 +876,14 @@ sf_buf_alloc(struct vm_page *m, int flag CPU_ZERO(&sf->cpumask); shootdown: sched_pin(); - cpumask = PCPU_GET(cpumask); - if (!CPU_OVERLAP(&cpumask, &sf->cpumask)) { - CPU_OR(&sf->cpumask, &cpumask); + cpuid = PCPU_GET(cpuid); + if (!CPU_ISSET(cpuid, &sf->cpumask)) { + CPU_SET(cpuid, &sf->cpumask); invlpg(sf->kva); } *** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201107041204.p64C4q8p018157>