0) { @@ -1527,7 +1535,7 @@ vm_phys_unfree_page(vm_paddr_t pa) m_tmp = m_set; m_set = vm_phys_seg_paddr_to_vm_page(seg, pa_half); } - vm_freelist_add(fl, m_tmp, order, 0); + vm_freelist_add(fl, m_tmp, order, pool, 0); } KASSERT(m_set == m, ("vm_phys_unfree_page: fatal inconsistency")); return (true); @@ -1668,7 +1676,8 @@ vm_phys_find_queues_contig( * alignment of the first physical page in the set. If the given value * "boundary" is non-zero, then the set of physical pages cannot cross * any physical address boundary that is a multiple of that value. Both - * "alignment" and "boundary" must be a power of two. + * "alignment" and "boundary" must be a power of two. Sets the pool + * field to DEFAULT in the first allocated page. */ vm_page_t vm_phys_alloc_contig(int domain, u_long npages, vm_paddr_t low, vm_paddr_t high, @@ -1727,12 +1736,12 @@ vm_phys_alloc_contig(int domain, u_long npages, vm_paddr_t low, vm_paddr_t high, fl = (*queues)[m->pool]; oind = m->order; vm_freelist_rem(fl, m, oind); - if (m->pool != VM_FREEPOOL_DEFAULT) - vm_phys_set_pool(VM_FREEPOOL_DEFAULT, m, oind); + vm_phys_finish_init(m, oind); } /* Return excess pages to the free lists. */ fl = (*queues)[VM_FREEPOOL_DEFAULT]; - vm_phys_enq_range(&m_run[npages], m - &m_run[npages], fl, 0); + vm_phys_enq_range(&m_run[npages], m - &m_run[npages], fl, + VM_FREEPOOL_DEFAULT, 0); /* Return page verified to satisfy conditions of request. */ pa_start = VM_PAGE_TO_PHYS(m_run); diff --git a/sys/vm/vm_phys.h b/sys/vm/vm_phys.h index 43d94a9420f2..046fe26b476d 100644 --- a/sys/vm/vm_phys.h +++ b/sys/vm/vm_phys.h @@ -64,15 +64,15 @@ vm_page_t vm_phys_alloc_contig(int domain, u_long npages, vm_paddr_t low, int vm_phys_alloc_npages(int domain, int pool, int npages, vm_page_t ma[]); vm_page_t vm_phys_alloc_pages(int domain, int pool, int order); int vm_phys_domain_match(int prefer, vm_paddr_t low, vm_paddr_t high); -void vm_phys_enqueue_contig(vm_page_t m, u_long npages); +void vm_phys_enqueue_contig(vm_page_t m, int pool, u_long npages); int vm_phys_fictitious_reg_range(vm_paddr_t start, vm_paddr_t end, vm_memattr_t memattr); void vm_phys_fictitious_unreg_range(vm_paddr_t start, vm_paddr_t end); vm_page_t vm_phys_fictitious_to_vm_page(vm_paddr_t pa); int vm_phys_find_range(vm_page_t bounds[], int segind, int domain, u_long npages, vm_paddr_t low, vm_paddr_t high); -void vm_phys_free_contig(vm_page_t m, u_long npages); -void vm_phys_free_pages(vm_page_t m, int order); +void vm_phys_free_contig(vm_page_t m, int pool, u_long npages); +void vm_phys_free_pages(vm_page_t m, int pool, int order); void vm_phys_init(void); vm_page_t vm_phys_paddr_to_vm_page(vm_paddr_t pa); vm_page_t vm_phys_seg_paddr_to_vm_page(struct vm_phys_seg *seg, vm_paddr_t pa); diff --git a/sys/vm/vm_reserv.c b/sys/vm/vm_reserv.c index ad513962d50d..04453e050a38 100644 --- a/sys/vm/vm_reserv.c +++ b/sys/vm/vm_reserv.c @@ -480,7 +480,7 @@ vm_reserv_depopulate(vm_reserv_t rv, int index) if (rv->popcnt == 0) { vm_reserv_remove(rv); vm_domain_free_lock(vmd); - vm_phys_free_pages(rv->pages, VM_LEVEL_0_ORDER); + vm_phys_free_pages(rv->pages, rv->pages->pool, VM_LEVEL_0_ORDER); vm_domain_free_unlock(vmd); counter_u64_add(vm_reserv_freed, 1); } @@ -943,7 +943,7 @@ static void vm_reserv_break(vm_reserv_t rv) { vm_page_t m; - int hi, lo, pos; + int pos, pos0, pos1; vm_reserv_assert_locked(rv); CTR5(KTR_VM, "%s: rv %p object %p popcnt %d inpartpop %d", @@ -954,23 +954,24 @@ vm_reserv_break(vm_reserv_t rv) for (; m < rv->pages + VM_LEVEL_0_NPAGES; m += VM_SUBLEVEL_0_NPAGES) #endif m->psind = 0; - hi = lo = -1; - pos = 0; - for (;;) { - bit_ff_at(rv->popmap, pos, VM_LEVEL_0_NPAGES, lo != hi, &pos); - if (lo == hi) { - if (pos == -1) - break; - lo = pos; - continue; - } + pos0 = bit_test(rv->popmap, 0) ? -1 : 0; + pos1 = -1 - pos0; + for (pos = 0; pos < VM_LEVEL_0_NPAGES; ) { + /* Find the first different bit after pos. */ + bit_ff_at(rv->popmap, pos + 1, VM_LEVEL_0_NPAGES, + pos1 < pos0, &pos); if (pos == -1) pos = VM_LEVEL_0_NPAGES; - hi = pos; + if (pos0 < pos1) { + pos0 = pos; + continue; + } + /* Free unused pages from pos0 to pos. */ + pos1 = pos; vm_domain_free_lock(VM_DOMAIN(rv->domain)); - vm_phys_enqueue_contig(&rv->pages[lo], hi - lo); + vm_phys_enqueue_contig(&rv->pages[pos0], VM_FREEPOOL_DEFAULT, + pos1 - pos0); vm_domain_free_unlock(VM_DOMAIN(rv->domain)); - lo = hi; } bit_nclear(rv->popmap, 0, VM_LEVEL_0_NPAGES - 1); rv->popcnt = 0;