Date: Sun, 29 Jan 2006 20:34:35 GMT From: Alan Cox <alc@FreeBSD.org> To: Perforce Change Reviews <perforce@freebsd.org> Subject: PERFORCE change 90637 for review Message-ID: <200601292034.k0TKYZB1092383@repoman.freebsd.org>
next in thread | raw e-mail | index | archive | help
http://perforce.freebsd.org/chv.cgi?CH=90637 Change 90637 by alc@alc_home on 2006/01/29 20:34:30 Convert PQ_CACHE to a collection of buddy queues. Revert PQ_BUDDY to its original name PQ_FREE. Affected files ... .. //depot/projects/superpages/src/sys/kern/vfs_bio.c#6 edit .. //depot/projects/superpages/src/sys/vm/vm_buddy.c#8 edit .. //depot/projects/superpages/src/sys/vm/vm_fault.c#6 edit .. //depot/projects/superpages/src/sys/vm/vm_map.c#5 edit .. //depot/projects/superpages/src/sys/vm/vm_object.c#7 edit .. //depot/projects/superpages/src/sys/vm/vm_page.c#14 edit .. //depot/projects/superpages/src/sys/vm/vm_page.h#8 edit .. //depot/projects/superpages/src/sys/vm/vm_pageout.c#8 edit .. //depot/projects/superpages/src/sys/vm/vm_pageq.c#16 edit .. //depot/projects/superpages/src/sys/vm/vm_zeroidle.c#5 edit Differences ... ==== //depot/projects/superpages/src/sys/kern/vfs_bio.c#6 (text+ko) ==== @@ -2864,7 +2864,7 @@ * page daemon? */ if ((curproc != pageproc) && - (VM_PAGE_INQUEUE1(m, PQ_CACHE)) && + (m->flags & PG_CACHE) != 0 && ((cnt.v_free_count + cnt.v_cache_count) < (cnt.v_free_min + cnt.v_cache_min))) { pagedaemon_wakeup(); ==== //depot/projects/superpages/src/sys/vm/vm_buddy.c#8 (text+ko) ==== @@ -67,7 +67,7 @@ for (q=0; q<BUDDY_QUEUES; q++) { vm_page_t p; - TAILQ_FOREACH(p, &vm_page_queues[PQ_BUDDY+q].pl, pageq) { + TAILQ_FOREACH(p, &vm_page_queues[PQ_FREE+q].pl, pageq) { for (i=0; i<(1<<q); i++) { vm_page_t m = PHYS_TO_VM_PAGE(p->phys_addr + i*PAGE_SIZE); INVARIANT(!m->reserv || m->flags & PG_CACHE); @@ -94,7 +94,7 @@ for (q = BUDDY_QUEUES - 1; q >= 0; q--) sbuf_printf(&sbuf, "%5.5d: %6.6dK, %6.6d\n", q, 1 << (PAGE_SHIFT - 10 + q), - vm_page_queues[PQ_BUDDY + q].lcnt); + vm_page_queues[PQ_FREE + q].lcnt); sbuf_finish(&sbuf); error = SYSCTL_OUT(req, sbuf_data(&sbuf), sbuf_len(&sbuf)); sbuf_delete(&sbuf); @@ -114,7 +114,7 @@ m, m->queue)); m->buddyq = q; m->queue = baseq + q; - if (baseq == PQ_CACHE || (m->flags & PG_ZERO) != 0) + if (baseq != PQ_FREE || (m->flags & PG_ZERO) != 0) TAILQ_INSERT_TAIL(&vm_page_queues[m->queue].pl, m, pageq); else TAILQ_INSERT_HEAD(&vm_page_queues[m->queue].pl, m, pageq); @@ -148,7 +148,7 @@ (1 << (PAGE_SHIFT + q)), ("vm_page_buddy_split: page %p is not page %p's %d buddy", m_buddy, m, q)); - buddy_insert(PQ_BUDDY, q, m_buddy); + buddy_insert(PQ_FREE, q, m_buddy); } } @@ -178,8 +178,8 @@ mtx_assert(&vm_page_queue_free_mtx, MA_OWNED); for (q = Q; q < BUDDY_QUEUES; q++) { - m = prefer_zero ? TAILQ_LAST(&vm_page_queues[PQ_BUDDY + q].pl, - pglist) : TAILQ_FIRST(&vm_page_queues[PQ_BUDDY + q].pl); + m = prefer_zero ? TAILQ_LAST(&vm_page_queues[PQ_FREE + q].pl, + pglist) : TAILQ_FIRST(&vm_page_queues[PQ_FREE + q].pl); if (m != NULL) { buddy_remove(m); vm_page_buddy_split(m, q, Q); @@ -212,7 +212,7 @@ { mtx_lock_spin(&vm_page_queue_free_mtx); - buddy_free_locked(PQ_BUDDY, m, q); + buddy_free_locked(PQ_FREE, m, q); mtx_unlock_spin(&vm_page_queue_free_mtx); } @@ -236,7 +236,7 @@ case PQ_CACHE: mtx_assert(&vm_page_queue_mtx, MA_OWNED); break; - case PQ_BUDDY: + case PQ_FREE: mtx_assert(&vm_page_queue_free_mtx, MA_OWNED); break; default: @@ -251,7 +251,7 @@ buddy = phys_to_vm_page(pa_buddy); if (buddy == NULL || buddy->buddyq != q || - (buddy->queue - buddy->buddyq) != baseq) + !VM_PAGE_INQUEUE1(buddy, baseq)) break; buddy_remove(buddy); q++; @@ -272,14 +272,20 @@ int q; mtx_assert(&vm_page_queue_mtx, MA_OWNED); - KASSERT(m->flags & PG_CACHE, ("xxx")); + KASSERT(m->flags & PG_CACHE, ("buddy_unfree: page %p isn't cached", m)); buddy = m; - for (q = 0; !VM_PAGE_INQUEUE1(buddy, PQ_CACHE) && - q < BUDDY_QUEUES; q++) { - buddy = PHYS_TO_VM_PAGE(m->phys_addr & + for (q = 0; !VM_PAGE_INQUEUE1(buddy, PQ_CACHE) && q < BUDDY_QUEUES;) { + q++; + buddy = phys_to_vm_page(m->phys_addr & (~(vm_paddr_t)0 << (PAGE_SHIFT + q))); + KASSERT(buddy != NULL, ("buddy_unfree: buddy page is NULL")); } - KASSERT(q == buddy->buddyq, ("xxx")); + KASSERT(q < BUDDY_QUEUES, + ("buddy_unfree: queue %d is out of range", q)); + KASSERT(buddy->buddyq >= q, + ("buddy_unfree: queue %d is less than buddy page %p's queue %d", + q, buddy, buddy->buddyq)); + q = buddy->buddyq; /* * m is in the free list as part of a chunk of size 1<<q whose @@ -292,16 +298,16 @@ q--; half = buddy->phys_addr ^ (1 << (PAGE_SHIFT + q)); if (m->phys_addr < half) - other = phys_to_vm_page(half); + other = PHYS_TO_VM_PAGE(half); else { other = buddy; - buddy = phys_to_vm_page(half); + buddy = PHYS_TO_VM_PAGE(half); } buddy_insert(PQ_CACHE, q, other); } cnt.v_cache_count--; - KASSERT(buddy == m, ("yyy")); - KASSERT(m->queue == PQ_NONE, ("xxx")); + KASSERT(buddy == m, ("buddy_unfree: yyy")); + KASSERT(m->queue == PQ_NONE, ("buddy_unfree: xxx")); } /* @@ -328,7 +334,7 @@ for (Q = 0; (1 << Q) < npages; Q++); mtx_lock_spin(&vm_page_queue_free_mtx); for (q = min(Q, BUDDY_QUEUES - 1); q < BUDDY_QUEUES; q++) { - TAILQ_FOREACH(m_ret, &vm_page_queues[PQ_BUDDY + q].pl, pageq) { + TAILQ_FOREACH(m_ret, &vm_page_queues[PQ_FREE + q].pl, pageq) { /* * Is the size of this allocation request larger than * the largest block size? @@ -349,7 +355,7 @@ m = phys_to_vm_page(pa); if (m == NULL || m->buddyq != BUDDY_QUEUES - 1 || - (m->queue - m->buddyq) != PQ_BUDDY) + (m->queue - m->buddyq) != PQ_FREE) break; } /* If not, continue to the next block. */ @@ -397,7 +403,7 @@ KASSERT(m->queue == PQ_NONE, ("vm_page_alloc_contig: page %p has unexpected queue %d", m, m->queue)); - buddy_free_locked(PQ_BUDDY, m, 0); + buddy_free_locked(PQ_FREE, m, 0); } mtx_unlock_spin(&vm_page_queue_free_mtx); return (m_ret); ==== //depot/projects/superpages/src/sys/vm/vm_fault.c#6 (text+ko) ==== @@ -1011,7 +1011,7 @@ (m->flags & (PG_BUSY | PG_FICTITIOUS)) == 0) { vm_page_lock_queues(); - if (VM_PAGE_INQUEUE1(m, PQ_CACHE)) + if ((m->flags & PG_CACHE) != 0) vm_page_deactivate(m); mpte = pmap_enter_quick(pmap, addr, m, entry->protection, mpte); ==== //depot/projects/superpages/src/sys/vm/vm_map.c#5 (text+ko) ==== @@ -1426,7 +1426,7 @@ are_queues_locked = TRUE; vm_page_lock_queues(); } - if (VM_PAGE_INQUEUE1(p, PQ_CACHE)) + if ((p->flags & PG_CACHE) != 0) vm_page_deactivate(p); mpte = pmap_enter_quick(map->pmap, addr + ptoa(tmpidx), p, prot, mpte); ==== //depot/projects/superpages/src/sys/vm/vm_object.c#7 (text+ko) ==== @@ -705,7 +705,7 @@ curgeneration = object->generation; p = vm_page_lookup(object, tscan); if (p == NULL || p->valid == 0 || - VM_PAGE_INQUEUE1(p, PQ_CACHE)) { + (p->flags & PG_CACHE) != 0) { if (--scanlimit == 0) break; ++tscan; @@ -794,7 +794,7 @@ if (((p->flags & PG_CLEANCHK) == 0) || (pi < tstart) || (pi >= tend) || (p->valid == 0) || - VM_PAGE_INQUEUE1(p, PQ_CACHE)) { + (p->flags & PG_CACHE) != 0) { vm_page_flag_clear(p, PG_CLEANCHK); continue; } @@ -872,7 +872,7 @@ (tp->flags & PG_CLEANCHK) == 0) || (tp->busy != 0)) break; - if (VM_PAGE_INQUEUE1(tp, PQ_CACHE)) { + if ((tp->flags & PG_CACHE) != 0) { vm_page_flag_clear(tp, PG_CLEANCHK); break; } @@ -900,7 +900,7 @@ (tp->flags & PG_CLEANCHK) == 0) || (tp->busy != 0)) break; - if (VM_PAGE_INQUEUE1(tp, PQ_CACHE)) { + if ((tp->flags & PG_CACHE) != 0) { vm_page_flag_clear(tp, PG_CLEANCHK); break; } ==== //depot/projects/superpages/src/sys/vm/vm_page.c#14 (text+ko) ==== @@ -465,9 +465,9 @@ void vm_page_dirty(vm_page_t m) { - KASSERT(VM_PAGE_GETKNOWNQUEUE1(m) != PQ_CACHE, + KASSERT((m->flags & PG_CACHE) == 0, ("vm_page_dirty: page in cache!")); - KASSERT((m->queue - m->buddyq) != PQ_BUDDY, + KASSERT(VM_PAGE_GETKNOWNQUEUE1(m) != PQ_FREE, ("vm_page_dirty: page is free!")); m->dirty = VM_PAGE_BITS_ALL; } @@ -715,7 +715,7 @@ vm_page_remove(m); vm_page_insert(m, new_object, new_pindex); - if (VM_PAGE_INQUEUE1(m, PQ_CACHE)) + if ((m->flags & PG_CACHE) != 0) vm_page_deactivate(m); vm_page_dirty(m); } @@ -967,7 +967,7 @@ mtx_assert(&vm_page_queue_mtx, MA_OWNED); if (VM_PAGE_GETKNOWNQUEUE2(m) != PQ_ACTIVE) { - if (VM_PAGE_INQUEUE1(m, PQ_CACHE)) + if ((m->flags & PG_CACHE) != 0) cnt.v_reactivated++; vm_pageq_remove(m); if (m->wire_count == 0 && (m->flags & PG_UNMANAGED) == 0) { @@ -1035,12 +1035,12 @@ ("vm_page_free_toq: freeing mapped page %p", m)); cnt.v_tfree++; - if (m->busy || ((m->queue - m->buddyq) == PQ_BUDDY)) { + if (m->busy || VM_PAGE_INQUEUE1(m, PQ_FREE)) { printf( "vm_page_free: pindex(%lu), busy(%d), PG_BUSY(%d), hold(%d)\n", (u_long)m->pindex, m->busy, (m->flags & PG_BUSY) ? 1 : 0, m->hold_count); - if ((m->queue - m->buddyq) == PQ_BUDDY) + if (VM_PAGE_INQUEUE1(m, PQ_FREE)) panic("vm_page_free: freeing free page"); else panic("vm_page_free: freeing busy page"); @@ -1090,7 +1090,7 @@ mtx_lock_spin(&vm_page_queue_free_mtx); if (m->flags & PG_ZERO) ++vm_page_zero_count; - buddy_free_locked(PQ_BUDDY, m, 0); + buddy_free_locked(PQ_FREE, m, 0); mtx_unlock_spin(&vm_page_queue_free_mtx); vm_page_free_wakeup(); } @@ -1233,7 +1233,7 @@ if (VM_PAGE_INQUEUE2(m, PQ_INACTIVE)) return; if (m->wire_count == 0 && (m->flags & PG_UNMANAGED) == 0) { - if (VM_PAGE_INQUEUE1(m, PQ_CACHE)) + if ((m->flags & PG_CACHE) != 0) cnt.v_reactivated++; vm_page_flag_clear(m, PG_WINATCFLS); vm_pageq_remove(m); @@ -1317,7 +1317,7 @@ printf("vm_page_cache: attempting to cache busy page\n"); return; } - if (VM_PAGE_INQUEUE1(m, PQ_CACHE)) + if ((m->flags & PG_CACHE) != 0) return; /* @@ -1330,6 +1330,7 @@ (long)m->pindex); } vm_pageq_remove_nowakeup(m); + vm_page_flag_set(m, PG_CACHE); vm_pageq_enqueue(PQ_CACHE, m); vm_page_free_wakeup(); } @@ -1370,7 +1371,7 @@ */ if ((dnw & 0x01F0) == 0 || VM_PAGE_INQUEUE2(m, PQ_INACTIVE) || - VM_PAGE_INQUEUE1(m, PQ_CACHE) + (m->flags & PG_CACHE) != 0 ) { if (m->act_count >= ACT_INIT) --m->act_count; @@ -1745,13 +1746,17 @@ DB_SHOW_COMMAND(pageq, vm_page_print_pageq_info) { int i; - db_printf("PQ_BUDDY:"); + db_printf("PQ_FREE:"); for (i = 0; i < BUDDY_QUEUES; i++) { - db_printf(" %d", vm_page_queues[PQ_BUDDY + i].lcnt); + db_printf(" %d", vm_page_queues[PQ_FREE + i].lcnt); } db_printf("\n"); - db_printf("PQ_CACHE: %d\n", vm_page_queues[PQ_CACHE].lcnt); + db_printf("PQ_CACHE:"); + for (i = 0; i < BUDDY_QUEUES; i++) { + db_printf(" %d", vm_page_queues[PQ_CACHE + i].lcnt); + } + db_printf("\n"); db_printf("PQ_ACTIVE: %d, PQ_INACTIVE: %d\n", vm_page_queues[PQ_ACTIVE].lcnt, ==== //depot/projects/superpages/src/sys/vm/vm_page.h#8 (text+ko) ==== @@ -145,29 +145,29 @@ /* PQ_CACHE and PQ_FREE represents a PQ_NUMCOLORS consecutive queue. */ #define PQ_NONE 0 -#define PQ_BUDDY 1 +#define PQ_FREE 1 #define PQ_INACTIVE (1 + BUDDY_QUEUES) #define PQ_ACTIVE (2 + BUDDY_QUEUES) #define PQ_CACHE (3 + BUDDY_QUEUES) -#define PQ_HOLD (4 + BUDDY_QUEUES) -#define PQ_COUNT (5 + BUDDY_QUEUES) +#define PQ_HOLD (3 + BUDDY_QUEUES*2) +#define PQ_COUNT (4 + BUDDY_QUEUES*2) /* Returns the real queue a page is on. */ #define VM_PAGE_GETQUEUE(m) ((m)->queue) /* Returns the well known queue a page is on. */ -#define VM_PAGE_GETKNOWNQUEUE1(m) ((m)->queue) +#define VM_PAGE_GETKNOWNQUEUE1(m) ((m)->queue - (m)->buddyq) #define VM_PAGE_GETKNOWNQUEUE2(m) VM_PAGE_GETQUEUE(m) /* Given the real queue number and a page color return the well know queue. */ -#define VM_PAGE_RESOLVEQUEUE(m, q) ((q)) +#define VM_PAGE_RESOLVEQUEUE(m, q) ((q) - (m)->buddyq) /* Returns true if the page is in the named well known queue. */ #define VM_PAGE_INQUEUE1(m, q) (VM_PAGE_GETKNOWNQUEUE1(m) == (q)) #define VM_PAGE_INQUEUE2(m, q) (VM_PAGE_GETKNOWNQUEUE2(m) == (q)) /* Sets the queue a page is on. */ -#define VM_PAGE_SETQUEUE1(m, q) (VM_PAGE_GETQUEUE(m) = (q)) +#define VM_PAGE_SETQUEUE1(m, q) (VM_PAGE_GETQUEUE(m) = (q) + (m)->buddyq) #define VM_PAGE_SETQUEUE2(m, q) (VM_PAGE_GETQUEUE(m) = (q)) struct vpgqueues { ==== //depot/projects/superpages/src/sys/vm/vm_pageout.c#8 (text+ko) ==== @@ -337,7 +337,7 @@ ib = 0; break; } - if (VM_PAGE_INQUEUE1(p, PQ_CACHE) || + if ((p->flags & PG_CACHE) != 0 || (p->flags & (PG_BUSY|PG_UNMANAGED)) || p->busy) { ib = 0; break; @@ -367,7 +367,7 @@ if ((p = vm_page_lookup(object, pindex + is)) == NULL) break; - if (VM_PAGE_INQUEUE1(p, PQ_CACHE) || + if ((p->flags & PG_CACHE) != 0 || (p->flags & (PG_BUSY|PG_UNMANAGED)) || p->busy) { break; } ==== //depot/projects/superpages/src/sys/vm/vm_pageq.c#16 (text+ko) ==== @@ -58,9 +58,11 @@ int i; for (i = 0; i < BUDDY_QUEUES; i++) { - vm_page_queues[PQ_BUDDY+i].cnt = &cnt.v_free_count; + vm_page_queues[PQ_FREE+i].cnt = &cnt.v_free_count; + } + for (i = 0; i < BUDDY_QUEUES; i++) { + vm_page_queues[PQ_CACHE + i].cnt = &cnt.v_cache_count; } - vm_page_queues[PQ_CACHE].cnt = &cnt.v_cache_count; vm_page_queues[PQ_INACTIVE].cnt = &cnt.v_inactive_count; vm_page_queues[PQ_ACTIVE].cnt = &cnt.v_active_count; vm_page_queues[PQ_HOLD].cnt = &cnt.v_active_count; @@ -204,9 +206,5 @@ TAILQ_REMOVE(&pq->pl, m, pageq); (*pq->cnt)--; pq->lcnt--; - if (VM_PAGE_RESOLVEQUEUE(m, queue) == PQ_CACHE) { - if (vm_paging_needed()) - pagedaemon_wakeup(); - } } } ==== //depot/projects/superpages/src/sys/vm/vm_zeroidle.c#5 (text+ko) ==== @@ -106,14 +106,14 @@ mtx_lock_spin(&vm_page_queue_free_mtx); zero_state = 0; - m = TAILQ_FIRST(&vm_page_queues[PQ_BUDDY].pl); + m = TAILQ_FIRST(&vm_page_queues[PQ_FREE].pl); if (m != NULL && (m->flags & PG_ZERO) == 0) { vm_pageq_remove_nowakeup(m); mtx_unlock_spin(&vm_page_queue_free_mtx); pmap_zero_page_idle(m); mtx_lock_spin(&vm_page_queue_free_mtx); m->flags |= PG_ZERO; - vm_pageq_enqueue(PQ_BUDDY, m); + vm_pageq_enqueue(PQ_FREE, m); ++vm_page_zero_count; ++cnt_prezero; if (vm_page_zero_count >= ZIDLE_HI(cnt.v_free_count))
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?200601292034.k0TKYZB1092383>