From owner-p4-projects@FreeBSD.ORG Tue Jan 22 23:49:15 2008 Return-Path: Delivered-To: p4-projects@freebsd.org Received: by hub.freebsd.org (Postfix, from userid 32767) id E4E3616A468; Tue, 22 Jan 2008 23:49:14 +0000 (UTC) Delivered-To: perforce@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:4f8:fff6::34]) by hub.freebsd.org (Postfix) with ESMTP id A9D1C16A41B for ; Tue, 22 Jan 2008 23:49:14 +0000 (UTC) (envelope-from kmacy@freebsd.org) Received: from repoman.freebsd.org (repoman.freebsd.org [IPv6:2001:4f8:fff6::29]) by mx1.freebsd.org (Postfix) with ESMTP id 9D5FB13C45A for ; Tue, 22 Jan 2008 23:49:14 +0000 (UTC) (envelope-from kmacy@freebsd.org) Received: from repoman.freebsd.org (localhost [127.0.0.1]) by repoman.freebsd.org (8.14.1/8.14.1) with ESMTP id m0MNnE1h073420 for ; Tue, 22 Jan 2008 23:49:14 GMT (envelope-from kmacy@freebsd.org) Received: (from perforce@localhost) by repoman.freebsd.org (8.14.1/8.14.1/Submit) id m0MNnEEo073417 for perforce@freebsd.org; Tue, 22 Jan 2008 23:49:14 GMT (envelope-from kmacy@freebsd.org) Date: Tue, 22 Jan 2008 23:49:14 GMT Message-Id: <200801222349.m0MNnEEo073417@repoman.freebsd.org> X-Authentication-Warning: repoman.freebsd.org: perforce set sender to kmacy@freebsd.org using -f From: Kip Macy To: Perforce Change Reviews Cc: Subject: PERFORCE change 133896 for review X-BeenThere: p4-projects@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: p4 projects tree changes List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Tue, 22 Jan 2008 23:49:15 -0000 http://perforce.freebsd.org/chv.cgi?CH=133896 Change 133896 by kmacy@pandemonium:kmacy:xen31 on 2008/01/22 23:48:37 - teach prefault to batch calls to pmap_enter_quick as well as reduce locking overhead - convert pmap_qenter and pmap_enter_quick_locked to use the update_va_mapping interface as that allows xen to use its own linear page tables Affected files ... .. //depot/projects/xen31/sys/i386/xen/pmap.c#27 edit .. //depot/projects/xen31/sys/vm/pmap.h#5 edit .. //depot/projects/xen31/sys/vm/vm_fault.c#3 edit Differences ... ==== //depot/projects/xen31/sys/i386/xen/pmap.c#27 (text+ko) ==== @@ -270,7 +270,7 @@ static void free_pv_entry(pmap_t pmap, pv_entry_t pv); static pv_entry_t get_pv_entry(pmap_t locked_pmap, int try); -static vm_page_t pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, +static vm_page_t pmap_enter_quick_locked(multicall_entry_t *mcl, pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, vm_page_t mpte); static int pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t sva, vm_page_t *free); @@ -1217,22 +1217,37 @@ void pmap_qenter(vm_offset_t sva, vm_page_t *ma, int count) { - pt_entry_t *endpte, oldpte, *pte; - - oldpte = 0; + pt_entry_t *endpte, *pte; + vm_paddr_t pa; + int mclcount = 0; + multicall_entry_t mcl[16]; + multicall_entry_t *mclp = mcl; + pte = vtopte(sva); endpte = pte + count; while (pte < endpte) { - oldpte |= *pte; - PT_SET_VA(pte, VM_PAGE_TO_PHYS(*ma) | pgeflag | PG_RW | PG_V, FALSE); + pa = xpmap_ptom(VM_PAGE_TO_PHYS(*ma)) | pgeflag | PG_RW | PG_V; + + mclp->op = __HYPERVISOR_update_va_mapping; + mclp->args[0] = sva; + mclp->args[1] = (uint32_t)(pa & 0xffffffff); + mclp->args[2] = (uint32_t)(pa >> 32); + mclp->args[3] = (*pte & PG_V) ? UVMF_INVLPG : 0; + + sva += PAGE_SIZE; pte++; ma++; + mclp++; + mclcount++; + if (mclcount == 16) { + HYPERVISOR_multicall(mcl, mclcount); + mclp = mcl; + mclcount = 0; + } } - if ((oldpte & PG_V) != 0) - pmap_invalidate_range(kernel_pmap, sva, sva + count * - PAGE_SIZE); - else - PT_UPDATES_FLUSH(); + if (mclcount) + HYPERVISOR_multicall(mcl, mclcount); + } /* @@ -2716,17 +2731,31 @@ { vm_page_t m, mpte; vm_pindex_t diff, psize; - + multicall_entry_t mcl[16]; + multicall_entry_t *mclpstart, *mclp; + int count = 0; + mclpstart = mclp = mcl; + VM_OBJECT_LOCK_ASSERT(m_start->object, MA_OWNED); psize = atop(end - start); + mpte = NULL; m = m_start; PMAP_LOCK(pmap); while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) { - mpte = pmap_enter_quick_locked(pmap, start + ptoa(diff), m, + mpte = pmap_enter_quick_locked(mclp, pmap, start + ptoa(diff), m, prot, mpte); m = TAILQ_NEXT(m, listq); + mclp++; + count++; + if (count == 16) { + HYPERVISOR_multicall(mclpstart, count); + mclp = mclpstart; + count = 0; + } } + if (count) + HYPERVISOR_multicall(mclpstart, count); PMAP_UNLOCK(pmap); } @@ -2742,14 +2771,39 @@ void pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot) { + multicall_entry_t mcl; + + PMAP_LOCK(pmap); + (void) pmap_enter_quick_locked(&mcl, pmap, va, m, prot, NULL); + HYPERVISOR_multicall(&mcl, 1); + PMAP_UNLOCK(pmap); +} +void +pmap_enter_quick_range(pmap_t pmap, vm_offset_t *addrs, vm_page_t *pages, vm_prot_t *prots, int count) +{ + int i, index = 0; + multicall_entry_t mcl[16]; + multicall_entry_t *mclp = mcl; + PMAP_LOCK(pmap); - (void) pmap_enter_quick_locked(pmap, va, m, prot, NULL); + for (i = 0; i < count; i++, pages++, prots++, addrs++) { + (void) pmap_enter_quick_locked(mclp, pmap, *addrs, *pages, *prots, NULL); + index++; + mclp++; + if (index == 16) { + HYPERVISOR_multicall(mcl, count); + mclp = mcl; + index = 0; + } + } + if (index) + HYPERVISOR_multicall(mcl, index); PMAP_UNLOCK(pmap); } static vm_page_t -pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, +pmap_enter_quick_locked(multicall_entry_t *mcl, pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, vm_page_t mpte) { pt_entry_t *pte; @@ -2845,6 +2899,7 @@ pa |= pg_nx; #endif +#if 0 /* * Now validate mapping with RO protection */ @@ -2852,6 +2907,21 @@ pte_store(pte, pa | PG_V | PG_U); else pte_store(pte, pa | PG_V | PG_U | PG_MANAGED); +#else + /* + * Now validate mapping with RO protection + */ + if (m->flags & (PG_FICTITIOUS|PG_UNMANAGED)) + pa = xpmap_ptom(pa | PG_V | PG_U); + else + pa = xpmap_ptom(pa | PG_V | PG_U | PG_MANAGED); + + mcl->op = __HYPERVISOR_update_va_mapping; + mcl->args[0] = va; + mcl->args[1] = (uint32_t)(pa & 0xffffffff); + mcl->args[2] = (uint32_t)(pa >> 32); + mcl->args[3] = 0; +#endif return mpte; } ==== //depot/projects/xen31/sys/vm/pmap.h#5 (text+ko) ==== @@ -99,6 +99,8 @@ vm_prot_t, boolean_t); void pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot); +void pmap_enter_quick_range(pmap_t pmap, vm_offset_t *addrs, vm_page_t *m, + vm_prot_t *prot, int count); void pmap_enter_object(pmap_t pmap, vm_offset_t start, vm_offset_t end, vm_page_t m_start, vm_prot_t prot); vm_paddr_t pmap_extract(pmap_t pmap, vm_offset_t va); ==== //depot/projects/xen31/sys/vm/vm_fault.c#3 (text+ko) ==== @@ -932,12 +932,15 @@ static void vm_fault_prefault(pmap_t pmap, vm_offset_t addra, vm_map_entry_t entry) { - int i; + int i, count; vm_offset_t addr, starta; vm_pindex_t pindex; vm_page_t m; - vm_object_t object; - + vm_object_t object, lobject; + vm_prot_t prots[PAGEORDER_SIZE]; + vm_page_t pages[PAGEORDER_SIZE]; + vm_offset_t addrs[PAGEORDER_SIZE]; + if (pmap != vmspace_pmap(curthread->td_proc->p_vmspace)) return; @@ -949,10 +952,16 @@ } else if (starta > addra) { starta = 0; } - + + for (i = 0; i < PAGEORDER_SIZE; i++) + pages[i] = NULL; + count = 0; + lobject = object; + VM_OBJECT_LOCK(lobject); for (i = 0; i < PAGEORDER_SIZE; i++) { - vm_object_t backing_object, lobject; + vm_object_t backing_object; + addr = addra + prefault_pageorder[i]; if (addr > addra + (PFFOR * PAGE_SIZE)) addr = 0; @@ -964,35 +973,45 @@ continue; pindex = ((addr - entry->start) + entry->offset) >> PAGE_SHIFT; - lobject = object; - VM_OBJECT_LOCK(lobject); while ((m = vm_page_lookup(lobject, pindex)) == NULL && lobject->type == OBJT_DEFAULT && (backing_object = lobject->backing_object) != NULL) { if (lobject->backing_object_offset & PAGE_MASK) break; pindex += lobject->backing_object_offset >> PAGE_SHIFT; + + if (count) { + vm_page_lock_queues(); + pmap_enter_quick_range(pmap, addrs, pages, prots, count); + vm_page_unlock_queues(); + } + count = 0; VM_OBJECT_LOCK(backing_object); VM_OBJECT_UNLOCK(lobject); lobject = backing_object; + } /* * give-up when a page is not in memory */ - if (m == NULL) { - VM_OBJECT_UNLOCK(lobject); + if (m == NULL) break; - } + if (((m->valid & VM_PAGE_BITS_ALL) == VM_PAGE_BITS_ALL) && (m->busy == 0) && (m->flags & PG_FICTITIOUS) == 0) { - - vm_page_lock_queues(); - pmap_enter_quick(pmap, addr, m, entry->protection); - vm_page_unlock_queues(); + pages[count] = m; + prots[count] = entry->protection; + addrs[count] = addr; + count++; } - VM_OBJECT_UNLOCK(lobject); + } + if (count) { + vm_page_lock_queues(); + pmap_enter_quick_range(pmap, addrs, pages, prots, count); + vm_page_unlock_queues(); } + VM_OBJECT_UNLOCK(lobject); } /*