From owner-svn-src-all@FreeBSD.ORG Sun May 9 16:55:43 2010 Return-Path: Delivered-To: svn-src-all@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [69.147.83.52]) by hub.freebsd.org (Postfix) with ESMTP id 4D287106566C; Sun, 9 May 2010 16:55:43 +0000 (UTC) (envelope-from alc@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [69.147.83.44]) by mx1.freebsd.org (Postfix) with ESMTP id 3CE488FC08; Sun, 9 May 2010 16:55:43 +0000 (UTC) Received: from svn.freebsd.org (localhost [127.0.0.1]) by svn.freebsd.org (8.14.3/8.14.3) with ESMTP id o49Gthdf091352; Sun, 9 May 2010 16:55:43 GMT (envelope-from alc@svn.freebsd.org) Received: (from alc@localhost) by svn.freebsd.org (8.14.3/8.14.3/Submit) id o49GthC9091349; Sun, 9 May 2010 16:55:43 GMT (envelope-from alc@svn.freebsd.org) Message-Id: <201005091655.o49GthC9091349@svn.freebsd.org> From: Alan Cox Date: Sun, 9 May 2010 16:55:43 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org X-SVN-Group: head MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cc: Subject: svn commit: r207823 - head/sys/vm X-BeenThere: svn-src-all@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: "SVN commit messages for the entire src tree \(except for " user" and " projects" \)" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Sun, 09 May 2010 16:55:43 -0000 Author: alc Date: Sun May 9 16:55:42 2010 New Revision: 207823 URL: http://svn.freebsd.org/changeset/base/207823 Log: Push down the acquisition of the page queues lock into vm_pageq_remove(). (This eliminates a surprising number of page queues lock acquisitions by vm_fault() because the page's queue is PQ_NONE and thus the page queues lock is not needed to remove the page from a queue.) Modified: head/sys/vm/vm_fault.c head/sys/vm/vm_page.c Modified: head/sys/vm/vm_fault.c ============================================================================== --- head/sys/vm/vm_fault.c Sun May 9 16:27:42 2010 (r207822) +++ head/sys/vm/vm_fault.c Sun May 9 16:55:42 2010 (r207823) @@ -363,9 +363,7 @@ RetryFault:; vm_object_deallocate(fs.first_object); goto RetryFault; } - vm_page_lock_queues(); vm_pageq_remove(fs.m); - vm_page_unlock_queues(); vm_page_unlock(fs.m); /* Modified: head/sys/vm/vm_page.c ============================================================================== --- head/sys/vm/vm_page.c Sun May 9 16:27:42 2010 (r207822) +++ head/sys/vm/vm_page.c Sun May 9 16:55:42 2010 (r207823) @@ -169,6 +169,7 @@ TUNABLE_INT("vm.boot_pages", &boot_pages SYSCTL_INT(_vm, OID_AUTO, boot_pages, CTLFLAG_RD, &boot_pages, 0, "number of pages allocated for bootstrapping the VM system"); +static void vm_page_queue_remove(int queue, vm_page_t m); static void vm_page_enqueue(int queue, vm_page_t m); /* Make sure that u_long is at least 64 bits when PAGE_SIZE is 32K. */ @@ -1328,24 +1329,43 @@ vm_page_requeue(vm_page_t m) } /* + * vm_page_queue_remove: + * + * Remove the given page from the specified queue. + * + * The page and page queues must be locked. + */ +static __inline void +vm_page_queue_remove(int queue, vm_page_t m) +{ + struct vpgqueues *pq; + + mtx_assert(&vm_page_queue_mtx, MA_OWNED); + vm_page_lock_assert(m, MA_OWNED); + pq = &vm_page_queues[queue]; + TAILQ_REMOVE(&pq->pl, m, pageq); + (*pq->cnt)--; +} + +/* * vm_pageq_remove: * * Remove a page from its queue. * - * The queue containing the given page must be locked. + * The given page must be locked. * This routine may not block. */ void vm_pageq_remove(vm_page_t m) { int queue = VM_PAGE_GETQUEUE(m); - struct vpgqueues *pq; + vm_page_lock_assert(m, MA_OWNED); if (queue != PQ_NONE) { + vm_page_lock_queues(); VM_PAGE_SETQUEUE2(m, PQ_NONE); - pq = &vm_page_queues[queue]; - TAILQ_REMOVE(&pq->pl, m, pageq); - (*pq->cnt)--; + vm_page_queue_remove(queue, m); + vm_page_unlock_queues(); } } @@ -1380,18 +1400,20 @@ vm_page_enqueue(int queue, vm_page_t m) void vm_page_activate(vm_page_t m) { + int queue; vm_page_lock_assert(m, MA_OWNED); - if (VM_PAGE_GETKNOWNQUEUE2(m) != PQ_ACTIVE) { + if ((queue = VM_PAGE_GETKNOWNQUEUE2(m)) != PQ_ACTIVE) { if (m->wire_count == 0 && (m->flags & PG_UNMANAGED) == 0) { if (m->act_count < ACT_INIT) m->act_count = ACT_INIT; vm_page_lock_queues(); - vm_pageq_remove(m); + if (queue != PQ_NONE) + vm_page_queue_remove(queue, m); vm_page_enqueue(PQ_ACTIVE, m); vm_page_unlock_queues(); } else - KASSERT(m->queue == PQ_NONE, + KASSERT(queue == PQ_NONE, ("vm_page_activate: wired page %p is queued", m)); } else { if (m->act_count < ACT_INIT) @@ -1472,11 +1494,8 @@ vm_page_free_toq(vm_page_t m) * callback routine until after we've put the page on the * appropriate free queue. */ - if (VM_PAGE_GETQUEUE(m) != PQ_NONE) { - vm_page_lock_queues(); + if ((m->flags & PG_UNMANAGED) == 0) vm_pageq_remove(m); - vm_page_unlock_queues(); - } vm_page_remove(m); /* @@ -1554,11 +1573,8 @@ vm_page_wire(vm_page_t m) if (m->flags & PG_FICTITIOUS) return; if (m->wire_count == 0) { - if ((m->flags & PG_UNMANAGED) == 0) { - vm_page_lock_queues(); + if ((m->flags & PG_UNMANAGED) == 0) vm_pageq_remove(m); - vm_page_unlock_queues(); - } atomic_add_int(&cnt.v_wire_count, 1); } m->wire_count++; @@ -1633,22 +1649,26 @@ vm_page_unwire(vm_page_t m, int activate static inline void _vm_page_deactivate(vm_page_t m, int athead) { + int queue; vm_page_lock_assert(m, MA_OWNED); /* * Ignore if already inactive. */ - if (VM_PAGE_INQUEUE2(m, PQ_INACTIVE)) + if ((queue = VM_PAGE_GETKNOWNQUEUE2(m)) == PQ_INACTIVE) return; if (m->wire_count == 0 && (m->flags & PG_UNMANAGED) == 0) { vm_page_lock_queues(); vm_page_flag_clear(m, PG_WINATCFLS); - vm_pageq_remove(m); + if (queue != PQ_NONE) + vm_page_queue_remove(queue, m); if (athead) - TAILQ_INSERT_HEAD(&vm_page_queues[PQ_INACTIVE].pl, m, pageq); + TAILQ_INSERT_HEAD(&vm_page_queues[PQ_INACTIVE].pl, m, + pageq); else - TAILQ_INSERT_TAIL(&vm_page_queues[PQ_INACTIVE].pl, m, pageq); + TAILQ_INSERT_TAIL(&vm_page_queues[PQ_INACTIVE].pl, m, + pageq); VM_PAGE_SETQUEUE2(m, PQ_INACTIVE); cnt.v_inactive_count++; vm_page_unlock_queues(); @@ -1751,11 +1771,7 @@ vm_page_cache(vm_page_t m) /* * Remove the page from the paging queues. */ - if (VM_PAGE_GETQUEUE(m) != PQ_NONE) { - vm_page_lock_queues(); - vm_pageq_remove(m); - vm_page_unlock_queues(); - } + vm_pageq_remove(m); /* * Remove the page from the object's collection of resident