Date: Fri, 4 Apr 2003 16:24:01 -0800 (PST) From: Peter Wemm <peter@FreeBSD.org> To: Perforce Change Reviews <perforce@freebsd.org> Subject: PERFORCE change 28088 for review Message-ID: <200304050024.h350O1d6078763@repoman.freebsd.org>
next in thread | raw e-mail | index | archive | help
http://perforce.freebsd.org/chv.cgi?CH=28088 Change 28088 by peter@peter_daintree on 2003/04/04 16:23:43 IFC i386_hammer branch Affected files ... .. //depot/projects/hammer/libexec/rtld-elf/x86_64/reloc.c#3 integrate .. //depot/projects/hammer/sys/x86_64/acpica/acpi_machdep.c#7 integrate .. //depot/projects/hammer/sys/x86_64/acpica/acpi_wakeup.c#6 integrate .. //depot/projects/hammer/sys/x86_64/include/bus_at386.h#3 integrate .. //depot/projects/hammer/sys/x86_64/include/md_var.h#10 integrate .. //depot/projects/hammer/sys/x86_64/isa/clock.c#8 integrate .. //depot/projects/hammer/sys/x86_64/isa/intr_machdep.c#8 integrate .. //depot/projects/hammer/sys/x86_64/isa/intr_machdep.h#4 integrate .. //depot/projects/hammer/sys/x86_64/isa/isa_dma.c#3 integrate .. //depot/projects/hammer/sys/x86_64/isa/npx.c#17 integrate .. //depot/projects/hammer/sys/x86_64/x86_64/busdma_machdep.c#4 integrate .. //depot/projects/hammer/sys/x86_64/x86_64/machdep.c#30 integrate .. //depot/projects/hammer/sys/x86_64/x86_64/mem.c#6 integrate .. //depot/projects/hammer/sys/x86_64/x86_64/pmap.c#10 integrate .. //depot/projects/hammer/sys/x86_64/x86_64/support.s#6 integrate .. //depot/projects/hammer/sys/x86_64/x86_64/swtch.s#7 integrate .. //depot/projects/hammer/sys/x86_64/x86_64/trap.c#16 integrate .. //depot/projects/hammer/sys/x86_64/x86_64/vm_machdep.c#9 integrate Differences ... ==== //depot/projects/hammer/libexec/rtld-elf/x86_64/reloc.c#3 (text+ko) ==== @@ -22,7 +22,7 @@ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * - * $FreeBSD: src/libexec/rtld-elf/i386/reloc.c,v 1.11 2002/11/18 22:08:49 tmm Exp $ + * $FreeBSD: src/libexec/rtld-elf/i386/reloc.c,v 1.12 2003/03/14 21:10:12 kan Exp $ */ /* @@ -125,8 +125,6 @@ cache = mmap(NULL, bytes, PROT_READ|PROT_WRITE, MAP_ANON, -1, 0); if (cache == MAP_FAILED) cache = NULL; - if (cache != NULL) - memset(cache, 0, bytes); rellim = (const Elf_Rel *) ((caddr_t) obj->rel + obj->relsize); for (rel = obj->rel; rel < rellim; rel++) { ==== //depot/projects/hammer/sys/x86_64/acpica/acpi_machdep.c#7 (text+ko) ==== @@ -23,7 +23,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $FreeBSD: src/sys/i386/acpica/acpi_machdep.c,v 1.7 2003/03/03 12:15:48 phk Exp $ + * $FreeBSD: src/sys/i386/acpica/acpi_machdep.c,v 1.8 2003/03/24 19:14:45 mdodd Exp $ */ #include <sys/param.h> ==== //depot/projects/hammer/sys/x86_64/acpica/acpi_wakeup.c#6 (text+ko) ==== @@ -24,7 +24,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $FreeBSD: src/sys/i386/acpica/acpi_wakeup.c,v 1.17 2002/12/04 18:40:39 alc Exp $ + * $FreeBSD: src/sys/i386/acpica/acpi_wakeup.c,v 1.18 2003/03/25 00:07:01 jake Exp $ */ #include <sys/param.h> #include <sys/bus.h> ==== //depot/projects/hammer/sys/x86_64/include/bus_at386.h#3 (text+ko) ==== @@ -67,7 +67,7 @@ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -/* $FreeBSD: src/sys/i386/include/bus_at386.h,v 1.21 2003/03/11 19:43:38 jake Exp $ */ +/* $FreeBSD: src/sys/i386/include/bus_at386.h,v 1.22 2003/03/30 05:24:52 jake Exp $ */ #ifndef _I386_BUS_AT386_H_ #define _I386_BUS_AT386_H_ @@ -92,15 +92,23 @@ /* * Bus address and size types */ -typedef u_int bus_addr_t; -typedef u_int bus_size_t; +#ifdef PAE +typedef uint64_t bus_addr_t; +#else +typedef uint32_t bus_addr_t; +#endif +typedef uint32_t bus_size_t; #define BUS_SPACE_MAXSIZE_24BIT 0xFFFFFF #define BUS_SPACE_MAXSIZE_32BIT 0xFFFFFFFF #define BUS_SPACE_MAXSIZE 0xFFFFFFFF #define BUS_SPACE_MAXADDR_24BIT 0xFFFFFF #define BUS_SPACE_MAXADDR_32BIT 0xFFFFFFFF +#ifdef PAE +#define BUS_SPACE_MAXADDR 0xFFFFFFFFFFFFFFFFULL +#else #define BUS_SPACE_MAXADDR 0xFFFFFFFF +#endif #define BUS_SPACE_UNRESTRICTED (~0) ==== //depot/projects/hammer/sys/x86_64/include/md_var.h#10 (text+ko) ==== @@ -26,7 +26,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $FreeBSD: src/sys/i386/include/md_var.h,v 1.59 2003/01/22 18:18:45 jhb Exp $ + * $FreeBSD: src/sys/i386/include/md_var.h,v 1.62 2003/04/04 17:29:54 des Exp $ */ #ifndef _MACHINE_MD_VAR_H_ @@ -36,6 +36,11 @@ * Miscellaneous machine-dependent declarations. */ +extern void (*bcopy_vector)(const void *from, void *to, size_t len); +extern void (*bzero_vector)(void *buf, size_t len); +extern int (*copyin_vector)(const void *udaddr, void *kaddr, size_t len); +extern int (*copyout_vector)(const void *kaddr, void *udaddr, size_t len); + extern long Maxmem; extern u_int atdevbase; /* offset in virtual memory of ISA io mem */ extern int busdma_swi_pending; @@ -50,9 +55,6 @@ extern char kstack[]; extern char sigcode[]; extern int szsigcode; -#ifdef COMPAT_FREEBSD4 -extern int szfreebsd4_sigcode; -#endif typedef void alias_for_inthand_t(u_int cs, u_int ef, u_int esp, u_int ss); struct thread; ==== //depot/projects/hammer/sys/x86_64/isa/clock.c#8 (text+ko) ==== @@ -34,7 +34,7 @@ * SUCH DAMAGE. * * from: @(#)clock.c 7.2 (Berkeley) 5/12/91 - * $FreeBSD: src/sys/i386/isa/clock.c,v 1.195 2003/02/11 11:43:25 phk Exp $ + * $FreeBSD: src/sys/i386/isa/clock.c,v 1.197 2003/03/24 19:14:46 mdodd Exp $ */ /* @@ -52,7 +52,6 @@ #include "opt_isa.h" #include <sys/param.h> -#include <sys/stdint.h> #include <sys/systm.h> #include <sys/bus.h> #include <sys/lock.h> ==== //depot/projects/hammer/sys/x86_64/isa/intr_machdep.c#8 (text+ko) ==== @@ -34,7 +34,7 @@ * SUCH DAMAGE. * * from: @(#)isa.c 7.2 (Berkeley) 5/13/91 - * $FreeBSD: src/sys/i386/isa/intr_machdep.c,v 1.73 2002/10/20 18:02:46 phk Exp $ + * $FreeBSD: src/sys/i386/isa/intr_machdep.c,v 1.74 2003/03/24 19:14:46 mdodd Exp $ */ #include "opt_auto_eoi.h" ==== //depot/projects/hammer/sys/x86_64/isa/intr_machdep.h#4 (text+ko) ==== @@ -30,7 +30,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $FreeBSD: src/sys/i386/isa/intr_machdep.h,v 1.42 2002/07/12 07:56:10 peter Exp $ + * $FreeBSD: src/sys/i386/isa/intr_machdep.h,v 1.43 2003/04/02 23:53:29 peter Exp $ */ #ifndef _I386_ISA_INTR_MACHDEP_H_ ==== //depot/projects/hammer/sys/x86_64/isa/isa_dma.c#3 (text+ko) ==== @@ -34,7 +34,7 @@ * SUCH DAMAGE. * * from: @(#)isa.c 7.2 (Berkeley) 5/13/91 - * $FreeBSD: src/sys/i386/isa/isa_dma.c,v 1.12 2002/04/29 07:43:14 peter Exp $ + * $FreeBSD: src/sys/i386/isa/isa_dma.c,v 1.13 2003/03/25 00:07:03 jake Exp $ */ /* @@ -215,7 +215,7 @@ void isa_dmastart(int flags, caddr_t addr, u_int nbytes, int chan) { - vm_offset_t phys; + vm_paddr_t phys; int waport; caddr_t newaddr; @@ -375,7 +375,8 @@ static int isa_dmarangecheck(caddr_t va, u_int length, int chan) { - vm_offset_t phys, priorpage = 0, endva; + vm_paddr_t phys, priorpage = 0; + vm_offset_t endva; u_int dma_pgmsk = (chan & 4) ? ~(128*1024-1) : ~(64*1024-1); GIANT_REQUIRED; ==== //depot/projects/hammer/sys/x86_64/isa/npx.c#17 (text+ko) ==== @@ -32,7 +32,7 @@ * SUCH DAMAGE. * * from: @(#)npx.c 7.2 (Berkeley) 5/12/91 - * $FreeBSD: src/sys/i386/isa/npx.c,v 1.137 2003/02/17 09:55:08 julian Exp $ + * $FreeBSD: src/sys/i386/isa/npx.c,v 1.140 2003/04/04 17:29:55 des Exp $ */ #include "opt_debug_npx.h" @@ -614,9 +614,8 @@ bzero(addr, sizeof(*addr)); return (_MC_FPOWNED_NONE); } - s = intr_disable(); - if (curthread == PCPU_GET(fpcurthread)) { + if (td == PCPU_GET(fpcurthread)) { fxsave(addr); intr_restore(s); return (_MC_FPOWNED_FPU); @@ -638,7 +637,7 @@ register_t s; s = intr_disable(); - if (curthread == PCPU_GET(fpcurthread)) { + if (td == PCPU_GET(fpcurthread)) { fxrstor(addr); intr_restore(s); } else { ==== //depot/projects/hammer/sys/x86_64/x86_64/busdma_machdep.c#4 (text+ko) ==== @@ -23,7 +23,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $FreeBSD: src/sys/i386/i386/busdma_machdep.c,v 1.34 2003/02/26 02:16:05 mux Exp $ + * $FreeBSD: src/sys/i386/i386/busdma_machdep.c,v 1.38 2003/03/25 00:07:02 jake Exp $ */ #include <sys/param.h> @@ -31,6 +31,7 @@ #include <sys/malloc.h> #include <sys/bus.h> #include <sys/interrupt.h> +#include <sys/kernel.h> #include <sys/lock.h> #include <sys/proc.h> #include <sys/mutex.h> @@ -41,6 +42,7 @@ #include <vm/vm_page.h> #include <vm/vm_map.h> +#include <machine/atomic.h> #include <machine/bus.h> #include <machine/md_var.h> @@ -95,13 +97,17 @@ static STAILQ_HEAD(, bus_dmamap) bounce_map_callbacklist; static struct bus_dmamap nobounce_dmamap; +static void init_bounce_pages(void *dummy); static int alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages); static int reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map); -static vm_offset_t add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, +static bus_addr_t add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, vm_offset_t vaddr, bus_size_t size); static void free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage); static __inline int run_filter(bus_dma_tag_t dmat, bus_addr_t paddr); +/* To protect all the the bounce pages related lists and data. */ +static struct mtx bounce_lock; + /* * Return true if a match is made. * @@ -152,8 +158,9 @@ newtag->parent = parent; newtag->alignment = alignment; newtag->boundary = boundary; - newtag->lowaddr = trunc_page((vm_offset_t)lowaddr) + (PAGE_SIZE - 1); - newtag->highaddr = trunc_page((vm_offset_t)highaddr) + (PAGE_SIZE - 1); + newtag->lowaddr = trunc_page((vm_paddr_t)lowaddr) + (PAGE_SIZE - 1); + newtag->highaddr = trunc_page((vm_paddr_t)highaddr) + + (PAGE_SIZE - 1); newtag->filter = filter; newtag->filterarg = filterarg; newtag->maxsize = maxsize; @@ -181,12 +188,12 @@ newtag->filterarg = parent->filterarg; newtag->parent = parent->parent; } - if (newtag->parent != NULL) { - parent->ref_count++; - } + if (newtag->parent != NULL) + atomic_add_int(&parent->ref_count, 1); } - if (newtag->lowaddr < ptoa(Maxmem) && (flags & BUS_DMA_ALLOCNOW) != 0) { + if (newtag->lowaddr < ptoa((vm_paddr_t)Maxmem) && + (flags & BUS_DMA_ALLOCNOW) != 0) { /* Must bounce */ if (lowaddr > bounce_lowaddr) { @@ -230,7 +237,7 @@ bus_dma_tag_t parent; parent = dmat->parent; - dmat->ref_count--; + atomic_subtract_int(&dmat->ref_count, 1); if (dmat->ref_count == 0) { free(dmat, M_DEVBUF); /* @@ -257,7 +264,7 @@ error = 0; - if (dmat->lowaddr < ptoa(Maxmem)) { + if (dmat->lowaddr < ptoa((vm_paddr_t)Maxmem)) { /* Must bounce */ int maxpages; @@ -339,7 +346,8 @@ /* If we succeed, no mapping/bouncing will be required */ *mapp = NULL; - if ((size <= PAGE_SIZE) && dmat->lowaddr >= ptoa(Maxmem)) { + if ((size <= PAGE_SIZE) && + dmat->lowaddr >= ptoa((vm_paddr_t)Maxmem)) { *vaddr = malloc(size, M_DEVBUF, (flags & BUS_DMA_NOWAIT) ? M_NOWAIT : M_WAITOK); } else { @@ -348,10 +356,12 @@ * and handles multi-seg allocations. Nobody is doing * multi-seg allocations yet though. */ + mtx_lock(&Giant); *vaddr = contigmalloc(size, M_DEVBUF, (flags & BUS_DMA_NOWAIT) ? M_NOWAIT : M_WAITOK, 0ul, dmat->lowaddr, dmat->alignment? dmat->alignment : 1ul, dmat->boundary); + mtx_unlock(&Giant); } if (*vaddr == NULL) return (ENOMEM); @@ -379,10 +389,13 @@ */ if (map != NULL) panic("bus_dmamem_free: Invalid map freed\n"); - if ((size <= PAGE_SIZE) && dmat->lowaddr >= ptoa(Maxmem)) + if ((size <= PAGE_SIZE) && dmat->lowaddr >= ptoa((vm_paddr_t)Maxmem)) free(vaddr, M_DEVBUF); - else + else { + mtx_lock(&Giant); contigfree(vaddr, size, M_DEVBUF); + mtx_unlock(&Giant); + } } void @@ -402,7 +415,7 @@ void *callback_arg, int flags) { vm_offset_t vaddr; - vm_offset_t paddr; + vm_paddr_t paddr; #ifdef __GNUC__ bus_dma_segment_t dm_segments[dmat->nsegments]; #else @@ -411,7 +424,7 @@ bus_dma_segment_t *sg; int seg; int error; - vm_offset_t nextpaddr; + vm_paddr_t nextpaddr; if (map == NULL) map = &nobounce_dmamap; @@ -421,7 +434,8 @@ * If we are being called during a callback, pagesneeded will * be non-zero, so we can avoid doing the work twice. */ - if (dmat->lowaddr < ptoa(Maxmem) && map->pagesneeded == 0) { + if (dmat->lowaddr < ptoa((vm_paddr_t)Maxmem) && + map->pagesneeded == 0) { vm_offset_t vendaddr; /* @@ -443,9 +457,7 @@ /* Reserve Necessary Bounce Pages */ if (map->pagesneeded != 0) { - int s; - - s = splhigh(); + mtx_lock(&bounce_lock); if (reserve_bounce_pages(dmat, map) != 0) { /* Queue us for resources */ @@ -456,11 +468,10 @@ map->callback_arg = callback_arg; STAILQ_INSERT_TAIL(&bounce_map_waitinglist, map, links); - splx(s); - + mtx_unlock(&bounce_lock); return (EINPROGRESS); } - splx(s); + mtx_unlock(&bounce_lock); } vaddr = (vm_offset_t)buf; @@ -619,7 +630,7 @@ #endif int nsegs, error; - KASSERT(dmat->lowaddr >= ptoa(Maxmem) || map != NULL, + KASSERT(dmat->lowaddr >= ptoa((vm_paddr_t)Maxmem) || map != NULL, ("bus_dmamap_load_mbuf: No support for bounce pages!")); KASSERT(m0->m_flags & M_PKTHDR, ("bus_dmamap_load_mbuf: no packet header")); @@ -675,7 +686,7 @@ struct iovec *iov; struct thread *td = NULL; - KASSERT(dmat->lowaddr >= ptoa(Maxmem) || map != NULL, + KASSERT(dmat->lowaddr >= ptoa((vm_paddr_t)Maxmem) || map != NULL, ("bus_dmamap_load_uio: No support for bounce pages!")); resid = uio->uio_resid; @@ -772,42 +783,52 @@ } } +static void +init_bounce_pages(void *dummy __unused) +{ + + free_bpages = 0; + reserved_bpages = 0; + active_bpages = 0; + total_bpages = 0; + STAILQ_INIT(&bounce_page_list); + STAILQ_INIT(&bounce_map_waitinglist); + STAILQ_INIT(&bounce_map_callbacklist); + mtx_init(&bounce_lock, "bounce pages lock", NULL, MTX_DEF); +} +SYSINIT(bpages, SI_SUB_LOCK, SI_ORDER_ANY, init_bounce_pages, NULL); + static int alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages) { int count; count = 0; - if (total_bpages == 0) { - STAILQ_INIT(&bounce_page_list); - STAILQ_INIT(&bounce_map_waitinglist); - STAILQ_INIT(&bounce_map_callbacklist); - } - while (numpages > 0) { struct bounce_page *bpage; - int s; bpage = (struct bounce_page *)malloc(sizeof(*bpage), M_DEVBUF, M_NOWAIT | M_ZERO); if (bpage == NULL) break; + mtx_lock(&Giant); bpage->vaddr = (vm_offset_t)contigmalloc(PAGE_SIZE, M_DEVBUF, M_NOWAIT, 0ul, dmat->lowaddr, PAGE_SIZE, 0); + mtx_unlock(&Giant); if (bpage->vaddr == 0) { free(bpage, M_DEVBUF); break; } bpage->busaddr = pmap_kextract(bpage->vaddr); - s = splhigh(); + mtx_lock(&bounce_lock); STAILQ_INSERT_TAIL(&bounce_page_list, bpage, links); total_bpages++; free_bpages++; - splx(s); + mtx_unlock(&bounce_lock); count++; numpages--; } @@ -819,6 +840,7 @@ { int pages; + mtx_assert(&bounce_lock, MA_OWNED); pages = MIN(free_bpages, map->pagesneeded - map->pagesreserved); free_bpages -= pages; reserved_bpages += pages; @@ -828,11 +850,10 @@ return (pages); } -static vm_offset_t +static bus_addr_t add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, vm_offset_t vaddr, bus_size_t size) { - int s; struct bounce_page *bpage; if (map->pagesneeded == 0) @@ -843,7 +864,7 @@ panic("add_bounce_page: map doesn't need any pages"); map->pagesreserved--; - s = splhigh(); + mtx_lock(&bounce_lock); bpage = STAILQ_FIRST(&bounce_page_list); if (bpage == NULL) panic("add_bounce_page: free page list is empty"); @@ -851,7 +872,7 @@ STAILQ_REMOVE_HEAD(&bounce_page_list, links); reserved_bpages--; active_bpages++; - splx(s); + mtx_unlock(&bounce_lock); bpage->datavaddr = vaddr; bpage->datacount = size; @@ -862,13 +883,12 @@ static void free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage) { - int s; struct bus_dmamap *map; bpage->datavaddr = 0; bpage->datacount = 0; - s = splhigh(); + mtx_lock(&bounce_lock); STAILQ_INSERT_HEAD(&bounce_page_list, bpage, links); free_bpages++; active_bpages--; @@ -881,22 +901,21 @@ swi_sched(vm_ih, 0); } } - splx(s); + mtx_unlock(&bounce_lock); } void busdma_swi(void) { - int s; struct bus_dmamap *map; - s = splhigh(); + mtx_lock(&bounce_lock); while ((map = STAILQ_FIRST(&bounce_map_callbacklist)) != NULL) { STAILQ_REMOVE_HEAD(&bounce_map_callbacklist, links); - splx(s); + mtx_unlock(&bounce_lock); bus_dmamap_load(map->dmat, map, map->buf, map->buflen, map->callback, map->callback_arg, /*flags*/0); - s = splhigh(); + mtx_lock(&bounce_lock); } - splx(s); + mtx_unlock(&bounce_lock); } ==== //depot/projects/hammer/sys/x86_64/x86_64/machdep.c#30 (text+ko) ==== @@ -35,7 +35,7 @@ * SUCH DAMAGE. * * from: @(#)machdep.c 7.4 (Berkeley) 6/3/91 - * $FreeBSD: src/sys/i386/i386/machdep.c,v 1.554 2003/02/20 05:35:52 peter Exp $ + * $FreeBSD: src/sys/i386/i386/machdep.c,v 1.561 2003/04/02 23:53:28 peter Exp $ */ #include "opt_atalk.h" @@ -48,6 +48,7 @@ #include "opt_maxmem.h" #include "opt_msgbuf.h" #include "opt_perfmon.h" +#include "opt_swtch.h" #include "opt_kstack_pages.h" #include <sys/param.h> @@ -140,11 +141,40 @@ vm_offset_t proc0kstack; /* address of proc 0 kstack space */ #if defined(SWTCH_OPTIM_STATS) -extern int swtch_optim_stats; +int stupid_switch; +SYSCTL_INT(_debug, OID_AUTO, stupid_switch, + CTLFLAG_RW, &stupid_switch, 0, ""); +int swtch_optim_stats; SYSCTL_INT(_debug, OID_AUTO, swtch_optim_stats, - CTLFLAG_RD, &swtch_optim_stats, 0, ""); + CTLFLAG_RW, &swtch_optim_stats, 0, ""); +int tlb_flush_count; SYSCTL_INT(_debug, OID_AUTO, tlb_flush_count, - CTLFLAG_RD, &tlb_flush_count, 0, ""); + CTLFLAG_RW, &tlb_flush_count, 0, ""); +int lazy_flush_count; +SYSCTL_INT(_debug, OID_AUTO, lazy_flush_count, + CTLFLAG_RW, &lazy_flush_count, 0, ""); +int lazy_flush_fixup; +SYSCTL_INT(_debug, OID_AUTO, lazy_flush_fixup, + CTLFLAG_RW, &lazy_flush_fixup, 0, ""); +#ifdef SMP +int lazy_flush_smpfixup; +SYSCTL_INT(_debug, OID_AUTO, lazy_flush_smpfixup, + CTLFLAG_RW, &lazy_flush_smpfixup, 0, ""); +int lazy_flush_smpipi; +SYSCTL_INT(_debug, OID_AUTO, lazy_flush_smpipi, + CTLFLAG_RW, &lazy_flush_smpipi, 0, ""); +int lazy_flush_smpbadcr3; +SYSCTL_INT(_debug, OID_AUTO, lazy_flush_smpbadcr3, + CTLFLAG_RW, &lazy_flush_smpbadcr3, 0, ""); +int lazy_flush_smpmiss; +SYSCTL_INT(_debug, OID_AUTO, lazy_flush_smpmiss, + CTLFLAG_RW, &lazy_flush_smpmiss, 0, ""); +#endif +#endif +#ifdef LAZY_SWITCH +int lazy_flush_enable = 1; +SYSCTL_INT(_debug, OID_AUTO, lazy_flush_enable, + CTLFLAG_RW, &lazy_flush_enable, 0, ""); #endif int cold = 1; @@ -156,7 +186,7 @@ long Maxmem = 0; -vm_offset_t phys_avail[10]; +vm_paddr_t phys_avail[10]; /* must be 2 less so 0 0 can signal end of chunks */ #define PHYS_AVAIL_ARRAY_END ((sizeof(phys_avail) / sizeof(vm_offset_t)) - 2) @@ -181,8 +211,8 @@ #ifdef PERFMON perfmon_init(); #endif - printf("real memory = %u (%u MB)\n", ptoa(Maxmem), - ptoa(Maxmem) / 1048576); + printf("real memory = %ju (%ju MB)\n", ptoa((uintmax_t)Maxmem), + ptoa((uintmax_t)Maxmem) / 1048576); /* * Display any holes after the first chunk of extended memory. */ @@ -191,19 +221,22 @@ printf("Physical memory chunk(s):\n"); for (indx = 0; phys_avail[indx + 1] != 0; indx += 2) { - unsigned int size1; + vm_paddr_t size; - size1 = phys_avail[indx + 1] - phys_avail[indx]; - printf("0x%08x - 0x%08x, %u bytes (%u pages)\n", - phys_avail[indx], phys_avail[indx + 1] - 1, size1, - size1 / PAGE_SIZE); + size = phys_avail[indx + 1] - phys_avail[indx]; + printf( + "0x%016jx - 0x%016jx, %ju bytes (%ju pages)\n", + (uintmax_t)phys_avail[indx], + (uintmax_t)phys_avail[indx + 1] - 1, + (uintmax_t)size, (uintmax_t)size / PAGE_SIZE); } } vm_ksubmap_init(&kmi); - printf("avail memory = %u (%u MB)\n", ptoa(cnt.v_free_count), - ptoa(cnt.v_free_count) / 1048576); + printf("avail memory = %ju (%ju MB)\n", + ptoa((uintmax_t)cnt.v_free_count), + ptoa((uintmax_t)cnt.v_free_count) / 1048576); /* * Set up buffers, so they can be used to read disk labels. @@ -498,7 +531,7 @@ /* Go back to user mode if both flags are set. */ if ((eflags & PSL_VIP) && (eflags & PSL_VIF)) - trapsignal(p, SIGBUS, 0); + trapsignal(td, SIGBUS, 0); if (vm86->vm86_has_vme) { eflags = (tf->tf_eflags & ~VME_USERCHANGE) | @@ -544,7 +577,7 @@ cs = ucp->uc_mcontext.mc_cs; if (!CS_SECURE(cs)) { printf("freebsd4_sigreturn: cs = 0x%x\n", cs); - trapsignal(p, SIGBUS, T_PROTFLT); + trapsignal(td, SIGBUS, T_PROTFLT); return (EINVAL); } @@ -559,9 +592,9 @@ p->p_sigstk.ss_flags &= ~SS_ONSTACK; #endif - p->p_sigmask = ucp->uc_sigmask; - SIG_CANTMASK(p->p_sigmask); - signotify(p); + td->td_sigmask = ucp->uc_sigmask; + SIG_CANTMASK(td->td_sigmask); + signotify(td); PROC_UNLOCK(p); return (EJUSTRETURN); } @@ -632,9 +665,9 @@ p->p_sigstk.ss_flags &= ~SS_ONSTACK; #endif - p->p_sigmask = ucp->uc_sigmask; - SIG_CANTMASK(p->p_sigmask); - signotify(p); + td->td_sigmask = ucp->uc_sigmask; + SIG_CANTMASK(td->td_sigmask); + signotify(td); PROC_UNLOCK(p); return (EJUSTRETURN); } @@ -671,6 +704,9 @@ * XXX we need to have a cpu mask of idle cpus and generate an IPI or * otherwise generate some sort of interrupt to wake up cpus sitting in HLT. * Then we can have our cake and eat it too. + * + * XXX I'm turning it on for SMP as well by default for now. It seems to + * help lock contention somewhat, and this is critical for HTT. -Peter */ static int cpu_idle_hlt = 1; SYSCTL_INT(_machdep, OID_AUTO, cpu_idle_hlt, CTLFLAG_RW, @@ -685,6 +721,12 @@ void cpu_idle(void) { + +#ifdef SMP + if (mp_grab_cpu_hlt()) + return; +#endif + if (cpu_idle_hlt) { disable_intr(); if (sched_runnable()) { @@ -1051,13 +1093,15 @@ * * Total memory size may be set by the kernel environment variable * hw.physmem or the compile-time define MAXMEM. + * + * XXX first should be vm_paddr_t. */ static void getmemsize(int first) { int i, physmap_idx, pa_indx; u_int basemem, extmem; - vm_offset_t pa, physmap[PHYSMAP_SIZE]; + vm_paddr_t pa, physmap[PHYSMAP_SIZE]; pt_entry_t *pte; char *cp; struct bios_smap *smap; @@ -1079,12 +1123,8 @@ physmap_idx = 0; do { if (boothowto & RB_VERBOSE) - printf("SMAP type=%02x base=%08x %08x len=%08x %08x\n", - smap->type, - *(u_int32_t *)((char *)&smap->base + 4), - (u_int32_t)smap->base, - *(u_int32_t *)((char *)&smap->length + 4), - (u_int32_t)smap->length); + printf("SMAP type=%02x base=%016llx len=%016llx\n", + smap->type, smap->base, smap->length); if (smap->type != 0x01) goto next_run; @@ -1092,11 +1132,13 @@ if (smap->length == 0) goto next_run; +#ifndef PAE if (smap->base >= 0xffffffff) { printf("%uK of memory above 4GB ignored\n", (u_int)(smap->length / 1024)); goto next_run; } +#endif for (i = 0; i <= physmap_idx; i += 2) { if (smap->base < physmap[i + 1]) { @@ -1227,7 +1269,7 @@ * extend the last memory segment to the new limit. */ if (atop(physmap[physmap_idx + 1]) < Maxmem) - physmap[physmap_idx + 1] = ptoa(Maxmem); + physmap[physmap_idx + 1] = ptoa((vm_paddr_t)Maxmem); /* call pmap initialization to make new kernel address space */ pmap_bootstrap(first, 0); @@ -1246,9 +1288,9 @@ * round up the start address and round down the end address. */ for (i = 0; i <= physmap_idx; i += 2) { - vm_offset_t end; + vm_paddr_t end; - end = ptoa(Maxmem); + end = ptoa((vm_paddr_t)Maxmem); if (physmap[i + 1] < end) end = trunc_page(physmap[i + 1]); for (pa = round_page(physmap[i]); pa < end; pa += PAGE_SIZE) { @@ -1529,7 +1571,11 @@ dblfault_tss.tss_esp2 = (int)&dblfault_stack[sizeof(dblfault_stack)]; dblfault_tss.tss_ss = dblfault_tss.tss_ss0 = dblfault_tss.tss_ss1 = dblfault_tss.tss_ss2 = GSEL(GDATA_SEL, SEL_KPL); +#ifdef PAE + dblfault_tss.tss_cr3 = (int)IdlePDPT; +#else dblfault_tss.tss_cr3 = (int)IdlePTD; +#endif dblfault_tss.tss_eip = (int)dblfault_handler; dblfault_tss.tss_eflags = PSL_KERNEL; dblfault_tss.tss_ds = dblfault_tss.tss_es = dblfault_tss.tss_fs = @@ -1571,7 +1617,11 @@ /* setup proc 0's pcb */ thread0.td_pcb->pcb_flags = 0; /* XXXKSE */ +#ifdef PAE + thread0.td_pcb->pcb_cr3 = (int)IdlePDPT; +#else thread0.td_pcb->pcb_cr3 = (int)IdlePTD; +#endif thread0.td_pcb->pcb_ext = 0; thread0.td_frame = &proc0_tf; } ==== //depot/projects/hammer/sys/x86_64/x86_64/mem.c#6 (text+ko) ==== @@ -38,7 +38,7 @@ * * from: Utah $Hdr: mem.c 1.13 89/10/08$ * from: @(#)mem.c 7.2 (Berkeley) 5/9/91 - * $FreeBSD: src/sys/i386/i386/mem.c,v 1.104 2003/03/03 12:15:49 phk Exp $ + * $FreeBSD: src/sys/i386/i386/mem.c,v 1.106 2003/04/03 23:44:35 jake Exp $ */ /* @@ -179,8 +179,6 @@ if (addr < (vm_offset_t)VADDR(PTDPTDI, 0)) return (EFAULT); - if (eaddr >= (vm_offset_t)VADDR(APTDPTDI, 0)) - return (EFAULT); for (; addr < eaddr; addr += PAGE_SIZE) if (pmap_extract(kernel_pmap, addr) == 0) return (EFAULT); @@ -211,7 +209,7 @@ * instead of going through read/write * \*******************************************************/ static int -memmmap(dev_t dev, vm_offset_t offset, vm_offset_t *paddr, int prot) +memmmap(dev_t dev, vm_offset_t offset, vm_paddr_t *paddr, int prot) { switch (minor(dev)) { ==== //depot/projects/hammer/sys/x86_64/x86_64/pmap.c#10 (text+ko) ==== @@ -39,7 +39,38 @@ * SUCH DAMAGE. * * from: @(#)pmap.c 7.7 (Berkeley) 5/12/91 - * $FreeBSD: src/sys/i386/i386/pmap.c,v 1.397 2003/03/12 07:38:37 jake Exp $ + * $FreeBSD: src/sys/i386/i386/pmap.c,v 1.403 2003/04/03 23:44:35 jake Exp $ + */ +/*- + * Copyright (c) 2003 Networks Associates Technology, Inc. + * All rights reserved. + * + * This software was developed for the FreeBSD Project by Jake Burkholder, + * Safeport Network Services, and Network Associates Laboratories, the + * Security Research Division of Network Associates, Inc. under + * DARPA/SPAWAR contract N66001-01-C-8035 ("CBOSS"), as part of the DARPA + * CHATS research program. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. */ /* @@ -71,6 +102,7 @@ #include "opt_pmap.h" #include "opt_msgbuf.h" #include "opt_kstack_pages.h" +#include "opt_swtch.h" #include <sys/param.h> #include <sys/systm.h> @@ -144,9 +176,12 @@ LIST_HEAD(pmaplist, pmap); static struct pmaplist allpmaps; static struct mtx allpmaps_lock; +#if defined(SMP) && defined(LAZY_SWITCH) +static struct mtx lazypmap_lock; +#endif -vm_offset_t avail_start; /* PA of first available physical page */ -vm_offset_t avail_end; /* PA of last available physical page */ +vm_paddr_t avail_start; /* PA of first available physical page */ +vm_paddr_t avail_end; /* PA of last available physical page */ vm_offset_t virtual_avail; /* VA of first avail page (after kernel bss) */ vm_offset_t virtual_end; /* VA of last avail page (end of kernel AS) */ static boolean_t pmap_initialized = FALSE; /* Has pmap_init completed? */ @@ -157,6 +192,10 @@ vm_offset_t kernel_vm_end; u_int32_t KERNend; /* Written in 32 bit mode */ +#ifdef PAE +static uma_zone_t pdptzone; +#endif + /* * Data for the pv entry allocation mechanism */ @@ -186,7 +225,6 @@ static pt_entry_t *PADDR1 = 0; static PMAP_INLINE void free_pv_entry(pv_entry_t pv); -static pt_entry_t *get_ptbase(pmap_t pmap); static pv_entry_t get_pv_entry(void); static void i386_protection_init(void); static __inline void pmap_changebit(vm_page_t m, int bit, boolean_t setem); @@ -203,11 +241,13 @@ static vm_page_t pmap_allocpte(pmap_t pmap, vm_offset_t va); static vm_page_t _pmap_allocpte(pmap_t pmap, unsigned ptepindex); -static pt_entry_t *pmap_pte_quick(pmap_t pmap, vm_offset_t va); static vm_page_t pmap_page_lookup(vm_object_t object, vm_pindex_t pindex); static int pmap_unuse_pt(pmap_t, vm_offset_t, vm_page_t); static vm_offset_t pmap_kmem_choose(vm_offset_t addr); -static void *pmap_allocf(uma_zone_t zone, int bytes, u_int8_t *flags, int wait); +static void *pmap_pv_allocf(uma_zone_t zone, int bytes, u_int8_t *flags, int wait); +#ifdef PAE +static void *pmap_pdpt_allocf(uma_zone_t zone, int bytes, u_int8_t *flags, int wait); +#endif static pd_entry_t pdir4mb; >>> TRUNCATED FOR MAIL (1000 lines) <<<
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?200304050024.h350O1d6078763>