From owner-p4-projects@FreeBSD.ORG Mon Sep 1 05:40:24 2008 Return-Path: Delivered-To: p4-projects@freebsd.org Received: by hub.freebsd.org (Postfix, from userid 32767) id 467B51065672; Mon, 1 Sep 2008 05:40:24 +0000 (UTC) Delivered-To: perforce@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:4f8:fff6::34]) by hub.freebsd.org (Postfix) with ESMTP id 0A35A106566C for ; Mon, 1 Sep 2008 05:40:24 +0000 (UTC) (envelope-from nwhitehorn@freebsd.org) Received: from repoman.freebsd.org (repoman.freebsd.org [IPv6:2001:4f8:fff6::29]) by mx1.freebsd.org (Postfix) with ESMTP id EED2B8FC1F for ; Mon, 1 Sep 2008 05:40:23 +0000 (UTC) (envelope-from nwhitehorn@freebsd.org) Received: from repoman.freebsd.org (localhost [127.0.0.1]) by repoman.freebsd.org (8.14.2/8.14.2) with ESMTP id m815eNrv086191 for ; Mon, 1 Sep 2008 05:40:23 GMT (envelope-from nwhitehorn@freebsd.org) Received: (from perforce@localhost) by repoman.freebsd.org (8.14.2/8.14.1/Submit) id m815eM2V086189 for perforce@freebsd.org; Mon, 1 Sep 2008 05:40:22 GMT (envelope-from nwhitehorn@freebsd.org) Date: Mon, 1 Sep 2008 05:40:22 GMT Message-Id: <200809010540.m815eM2V086189@repoman.freebsd.org> X-Authentication-Warning: repoman.freebsd.org: perforce set sender to nwhitehorn@freebsd.org using -f From: Nathan Whitehorn To: Perforce Change Reviews Cc: Subject: PERFORCE change 148968 for review X-BeenThere: p4-projects@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: p4 projects tree changes List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Mon, 01 Sep 2008 05:40:24 -0000 http://perforce.freebsd.org/chv.cgi?CH=148968 Change 148968 by nwhitehorn@nwhitehorn_trantor on 2008/09/01 05:39:38 Fix WITNESS builds for good. This sacrifices some of our KVA space, and there are still some LORs, but my machine will now boot multiuser with WITNESS and INVARIANTS enabled. This is also my first G5 kernel build built on the G5... Affected files ... .. //depot/projects/ppc-g5/sys/powerpc/aim/mmu_oea64.c#4 edit Differences ... ==== //depot/projects/ppc-g5/sys/powerpc/aim/mmu_oea64.c#4 (text+ko) ==== @@ -193,20 +193,19 @@ SYNC(); -#if 0 +#if 1 /* * CPU documentation says that tlbie takes the VPN, not the - * VA. However, my attempts to compute the VPN don't work, and - * all implementations I have seen pass tlbie the VA, so disable - * this for now. + * VA. I think the code below does this correctly. We will see. */ - vpn = (uint64_t)((va & ADDR_PIDX) >> ADDR_PIDX_SHFT); + vpn = (uint64_t)(va & ADDR_PIDX); if (pmap != NULL) vpn |= ((uint64_t)(va_to_sr(pmap->pm_sr,va) & SR_VSID_MASK) - << 16); + << 28); +#else + vpn = va; #endif - vpn = va; vpn_hi = (uint32_t)(vpn >> 32); vpn_lo = (uint32_t)vpn; @@ -305,6 +304,9 @@ uma_zone_t moea64_upvo_zone; /* zone for pvo entries for unmanaged pages */ uma_zone_t moea64_mpvo_zone; /* zone for pvo entries for managed pages */ +vm_offset_t pvo_allocator_start; +vm_offset_t pvo_allocator_end; + #define BPVO_POOL_SIZE 327680 static struct pvo_entry *moea64_bpvo_pool; static int moea64_bpvo_pool_index = 0; @@ -352,7 +354,7 @@ * PVO calls. */ static int moea64_pvo_enter(pmap_t, uma_zone_t, struct pvo_head *, - vm_offset_t, vm_offset_t, uint64_t, int); + vm_offset_t, vm_offset_t, uint64_t, int, int); static void moea64_pvo_remove(struct pvo_entry *, int); static struct pvo_entry *moea64_pvo_find_va(pmap_t, vm_offset_t, int *); static struct lpte *moea64_pvo_to_pte(const struct pvo_entry *, int); @@ -971,6 +973,14 @@ virtual_end = VM_MAX_KERNEL_ADDRESS; /* + * Allocate some stupid buffer regions. + */ + + pvo_allocator_start = virtual_avail; + virtual_avail += SEGMENT_LENGTH/4; + pvo_allocator_end = virtual_avail; + + /* * Allocate some things for page zeroing */ @@ -1123,10 +1133,9 @@ moea64_scratchpage_pte[which]->pte_lo = moea64_scratchpage_pvo[which]->pvo_pte.lpte.pte_lo; + moea64_scratchpage_pte[which]->pte_hi |= LPTE_VALID; - EIEIO(); - TLBIE(kernel_pmap, moea64_scratchpage_va[which]); } @@ -1253,7 +1262,7 @@ pvo_flags |= PVO_FAKE; error = moea64_pvo_enter(pmap, zone, pvo_head, va, VM_PAGE_TO_PHYS(m), - pte_lo, pvo_flags); + pte_lo, pvo_flags, 0); if (pmap == kernel_pmap) TLBIE(pmap, va); @@ -1383,6 +1392,66 @@ return (m); } +static void * +moea64_uma_page_alloc(uma_zone_t zone, int bytes, u_int8_t *flags, int wait) +{ + /* + * This entire routine is a horrible hack to avoid bothering kmem + * for new KVA addresses. Because this can get called from inside + * kmem allocation routines, calling kmem for a new address here + * can lead to multiply locking non-recursive mutexes. + */ + static vm_pindex_t color; + vm_offset_t va; + + vm_page_t m; + int pflags, needed_lock; + + *flags = UMA_SLAB_PRIV; + needed_lock = !PMAP_LOCKED(kernel_pmap); + + if (needed_lock) + PMAP_LOCK(kernel_pmap); + + if ((wait & (M_NOWAIT|M_USE_RESERVE)) == M_NOWAIT) + pflags = VM_ALLOC_INTERRUPT | VM_ALLOC_WIRED; + else + pflags = VM_ALLOC_SYSTEM | VM_ALLOC_WIRED; + if (wait & M_ZERO) + pflags |= VM_ALLOC_ZERO; + + for (;;) { + m = vm_page_alloc(NULL, color++, pflags | VM_ALLOC_NOOBJ); + if (m == NULL) { + if (wait & M_NOWAIT) + return (NULL); + VM_WAIT; + } else + break; + } + + va = pvo_allocator_start; + pvo_allocator_start += PAGE_SIZE; + + if (pvo_allocator_start >= pvo_allocator_end) + panic("Ran out of PVO allocator buffer space!"); + + /* Now call pvo_enter in recursive mode */ + moea64_pvo_enter(kernel_pmap, moea64_upvo_zone, + &moea64_pvo_kunmanaged, va, VM_PAGE_TO_PHYS(m), LPTE_M, + PVO_WIRED | PVO_BOOTSTRAP, 1); + + TLBIE(kernel_pmap, va); + + if (needed_lock) + PMAP_UNLOCK(kernel_pmap); + + if ((wait & M_ZERO) && (m->flags & PG_ZERO) == 0) + bzero((void *)va, PAGE_SIZE); + + return (void *)va; +} + void moea64_init(mmu_t mmu) { @@ -1395,6 +1464,12 @@ moea64_mpvo_zone = uma_zcreate("MPVO entry", sizeof(struct pvo_entry), NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM | UMA_ZONE_NOFREE); + + if (!hw_direct_map) { + uma_zone_set_allocf(moea64_upvo_zone,moea64_uma_page_alloc); + uma_zone_set_allocf(moea64_mpvo_zone,moea64_uma_page_alloc); + } + moea64_initialized = TRUE; } @@ -1518,7 +1593,7 @@ PMAP_LOCK(kernel_pmap); error = moea64_pvo_enter(kernel_pmap, moea64_upvo_zone, &moea64_pvo_kunmanaged, va, pa, pte_lo, - PVO_WIRED | VM_PROT_EXECUTE); + PVO_WIRED | VM_PROT_EXECUTE, 0); TLBIE(kernel_pmap, va); @@ -1641,7 +1716,6 @@ int i, mask; u_int entropy; - KASSERT((int)pmap < VM_MIN_KERNEL_ADDRESS, ("moea64_pinit: virt pmap")); PMAP_LOCK_INIT(pmap); entropy = 0; @@ -1914,7 +1988,7 @@ static int moea64_pvo_enter(pmap_t pm, uma_zone_t zone, struct pvo_head *pvo_head, - vm_offset_t va, vm_offset_t pa, uint64_t pte_lo, int flags) + vm_offset_t va, vm_offset_t pa, uint64_t pte_lo, int flags, int recurse) { struct pvo_entry *pvo; u_int sr; @@ -1934,7 +2008,10 @@ moea64_pvo_enter_calls++; first = 0; - bootstrap = 0; + bootstrap = (flags & PVO_BOOTSTRAP); + + if (!moea64_initialized) + bootstrap = 1; /* * Compute the PTE Group index. @@ -1947,14 +2024,16 @@ * Remove any existing mapping for this page. Reuse the pvo entry if * there is a mapping. */ - LOCK_TABLE(); + if (!recurse) + LOCK_TABLE(); LIST_FOREACH(pvo, &moea64_pvo_table[ptegidx], pvo_olink) { if (pvo->pvo_pmap == pm && PVO_VADDR(pvo) == va) { if ((pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN) == pa && (pvo->pvo_pte.lpte.pte_lo & LPTE_PP) == (pte_lo & LPTE_PP)) { - UNLOCK_TABLE(); + if (!recurse) + UNLOCK_TABLE(); return (0); } moea64_pvo_remove(pvo, -1); @@ -1965,9 +2044,7 @@ /* * If we aren't overwriting a mapping, try to allocate. */ - if (moea64_initialized && !mtx_recursed(&moea64_table_mutex)) { - pvo = uma_zalloc(zone, M_NOWAIT); - } else { + if (bootstrap) { if (moea64_bpvo_pool_index >= BPVO_POOL_SIZE) { panic("moea64_enter: bpvo pool exhausted, %d, %d, %d", moea64_bpvo_pool_index, BPVO_POOL_SIZE, @@ -1976,10 +2053,13 @@ pvo = &moea64_bpvo_pool[moea64_bpvo_pool_index]; moea64_bpvo_pool_index++; bootstrap = 1; + } else { + pvo = uma_zalloc(zone, M_NOWAIT); } if (pvo == NULL) { - UNLOCK_TABLE(); + if (!recurse) + UNLOCK_TABLE(); return (ENOMEM); } @@ -2024,7 +2104,9 @@ panic("moea64_pvo_enter: overflow"); moea64_pte_overflow++; } - UNLOCK_TABLE(); + + if (!recurse) + UNLOCK_TABLE(); return (first ? ENOENT : 0); }