Date: Mon, 2 Jul 2012 17:22:38 +0000 (UTC) From: Alan Cox <alc@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-stable@freebsd.org, svn-src-stable-9@freebsd.org Subject: svn commit: r238005 - in stable/9/sys: amd64/amd64 i386/i386 i386/xen Message-ID: <201207021722.q62HMcaJ083799@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: alc Date: Mon Jul 2 17:22:38 2012 New Revision: 238005 URL: http://svn.freebsd.org/changeset/base/238005 Log: MFC r236534 Various small changes to PV entry management. Modified: stable/9/sys/amd64/amd64/pmap.c stable/9/sys/i386/i386/pmap.c stable/9/sys/i386/xen/pmap.c Directory Properties: stable/9/sys/ (props changed) Modified: stable/9/sys/amd64/amd64/pmap.c ============================================================================== --- stable/9/sys/amd64/amd64/pmap.c Mon Jul 2 16:57:13 2012 (r238004) +++ stable/9/sys/amd64/amd64/pmap.c Mon Jul 2 17:22:38 2012 (r238005) @@ -2021,7 +2021,7 @@ pv_to_chunk(pv_entry_t pv) #define PC_FREE1 0xfffffffffffffffful #define PC_FREE2 0x000000fffffffffful -static uint64_t pc_freemask[_NPCM] = { PC_FREE0, PC_FREE1, PC_FREE2 }; +static const uint64_t pc_freemask[_NPCM] = { PC_FREE0, PC_FREE1, PC_FREE2 }; SYSCTL_LONG(_vm_pmap, OID_AUTO, pv_entry_count, CTLFLAG_RD, &pv_entry_count, 0, "Current number of pv entries"); @@ -2070,7 +2070,7 @@ pmap_pv_reclaim(pmap_t locked_pmap) pv_entry_t pv; vm_offset_t va; vm_page_t free, m, m_pc; - uint64_t inuse, freemask; + uint64_t inuse; int bit, field, freed; rw_assert(&pvh_global_lock, RA_WLOCKED); @@ -2102,7 +2102,6 @@ pmap_pv_reclaim(pmap_t locked_pmap) */ freed = 0; for (field = 0; field < _NPCM; field++) { - freemask = 0; for (inuse = ~pc->pc_map[field] & pc_freemask[field]; inuse != 0; inuse &= ~(1UL << bit)) { bit = bsfq(inuse); @@ -2131,16 +2130,16 @@ pmap_pv_reclaim(pmap_t locked_pmap) PGA_WRITEABLE); } } + pc->pc_map[field] |= 1UL << bit; pmap_unuse_pt(pmap, va, *pde, &free); - freemask |= 1UL << bit; freed++; } - pc->pc_map[field] |= freemask; } if (freed == 0) { TAILQ_INSERT_TAIL(&newtail, pc, pc_lru); continue; } + /* Every freed mapping is for a 4 KB page. */ pmap_resident_count_dec(pmap, freed); PV_STAT(pv_entry_frees += freed); PV_STAT(pv_entry_spare += freed); @@ -2261,10 +2260,6 @@ retry: TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list); } - if (pc != TAILQ_LAST(&pv_chunks, pch)) { - TAILQ_REMOVE(&pv_chunks, pc, pc_lru); - TAILQ_INSERT_TAIL(&pv_chunks, pc, pc_lru); - } pv_entry_count++; PV_STAT(pv_entry_spare--); return (pv); @@ -4138,7 +4133,7 @@ pmap_remove_pages(pmap_t pmap) TAILQ_FOREACH_SAFE(pc, &pmap->pm_pvchunk, pc_list, npc) { allfree = 1; for (field = 0; field < _NPCM; field++) { - inuse = (~(pc->pc_map[field])) & pc_freemask[field]; + inuse = ~pc->pc_map[field] & pc_freemask[field]; while (inuse != 0) { bit = bsfq(inuse); bitmask = 1UL << bit; Modified: stable/9/sys/i386/i386/pmap.c ============================================================================== --- stable/9/sys/i386/i386/pmap.c Mon Jul 2 16:57:13 2012 (r238004) +++ stable/9/sys/i386/i386/pmap.c Mon Jul 2 17:22:38 2012 (r238005) @@ -2177,7 +2177,7 @@ pv_to_chunk(pv_entry_t pv) #define PC_FREE0_9 0xfffffffful /* Free values for index 0 through 9 */ #define PC_FREE10 0x0000fffful /* Free values for index 10 */ -static uint32_t pc_freemask[_NPCM] = { +static const uint32_t pc_freemask[_NPCM] = { PC_FREE0_9, PC_FREE0_9, PC_FREE0_9, PC_FREE0_9, PC_FREE0_9, PC_FREE0_9, PC_FREE0_9, PC_FREE0_9, PC_FREE0_9, @@ -2227,7 +2227,7 @@ pmap_pv_reclaim(pmap_t locked_pmap) pv_entry_t pv; vm_offset_t va; vm_page_t free, m, m_pc; - uint32_t inuse, freemask; + uint32_t inuse; int bit, field, freed; PMAP_LOCK_ASSERT(locked_pmap, MA_OWNED); @@ -2260,7 +2260,6 @@ pmap_pv_reclaim(pmap_t locked_pmap) */ freed = 0; for (field = 0; field < _NPCM; field++) { - freemask = 0; for (inuse = ~pc->pc_map[field] & pc_freemask[field]; inuse != 0; inuse &= ~(1UL << bit)) { bit = bsfl(inuse); @@ -2289,16 +2288,16 @@ pmap_pv_reclaim(pmap_t locked_pmap) PGA_WRITEABLE); } } + pc->pc_map[field] |= 1UL << bit; pmap_unuse_pt(pmap, va, &free); - freemask |= 1UL << bit; freed++; } - pc->pc_map[field] |= freemask; } if (freed == 0) { TAILQ_INSERT_TAIL(&newtail, pc, pc_lru); continue; } + /* Every freed mapping is for a 4 KB page. */ pmap->pm_stats.resident_count -= freed; PV_STAT(pv_entry_frees += freed); PV_STAT(pv_entry_spare += freed); @@ -2367,13 +2366,21 @@ free_pv_entry(pmap_t pmap, pv_entry_t pv field = idx / 32; bit = idx % 32; pc->pc_map[field] |= 1ul << bit; - /* move to head of list */ - TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); for (idx = 0; idx < _NPCM; idx++) if (pc->pc_map[idx] != pc_freemask[idx]) { - TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list); + /* + * 98% of the time, pc is already at the head of the + * list. If it isn't already, move it to the head. + */ + if (__predict_false(TAILQ_FIRST(&pmap->pm_pvchunk) != + pc)) { + TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); + TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, + pc_list); + } return; } + TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); free_pv_chunk(pc); } @@ -2437,10 +2444,6 @@ retry: } TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list); - if (pc != TAILQ_LAST(&pv_chunks, pch)) { - TAILQ_REMOVE(&pv_chunks, pc, pc_lru); - TAILQ_INSERT_TAIL(&pv_chunks, pc, pc_lru); - } PV_STAT(pv_entry_spare--); return (pv); } @@ -4371,7 +4374,7 @@ pmap_remove_pages(pmap_t pmap) TAILQ_FOREACH_SAFE(pc, &pmap->pm_pvchunk, pc_list, npc) { allfree = 1; for (field = 0; field < _NPCM; field++) { - inuse = (~(pc->pc_map[field])) & pc_freemask[field]; + inuse = ~pc->pc_map[field] & pc_freemask[field]; while (inuse != 0) { bit = bsfl(inuse); bitmask = 1UL << bit; Modified: stable/9/sys/i386/xen/pmap.c ============================================================================== --- stable/9/sys/i386/xen/pmap.c Mon Jul 2 16:57:13 2012 (r238004) +++ stable/9/sys/i386/xen/pmap.c Mon Jul 2 17:22:38 2012 (r238005) @@ -1930,7 +1930,7 @@ pv_to_chunk(pv_entry_t pv) #define PC_FREE0_9 0xfffffffful /* Free values for index 0 through 9 */ #define PC_FREE10 0x0000fffful /* Free values for index 10 */ -static uint32_t pc_freemask[_NPCM] = { +static const uint32_t pc_freemask[_NPCM] = { PC_FREE0_9, PC_FREE0_9, PC_FREE0_9, PC_FREE0_9, PC_FREE0_9, PC_FREE0_9, PC_FREE0_9, PC_FREE0_9, PC_FREE0_9, @@ -1978,7 +1978,7 @@ pmap_pv_reclaim(pmap_t locked_pmap) pv_entry_t pv; vm_offset_t va; vm_page_t free, m, m_pc; - uint32_t inuse, freemask; + uint32_t inuse; int bit, field, freed; PMAP_LOCK_ASSERT(locked_pmap, MA_OWNED); @@ -2011,7 +2011,6 @@ pmap_pv_reclaim(pmap_t locked_pmap) */ freed = 0; for (field = 0; field < _NPCM; field++) { - freemask = 0; for (inuse = ~pc->pc_map[field] & pc_freemask[field]; inuse != 0; inuse &= ~(1UL << bit)) { bit = bsfl(inuse); @@ -2031,16 +2030,16 @@ pmap_pv_reclaim(pmap_t locked_pmap) TAILQ_REMOVE(&m->md.pv_list, pv, pv_list); if (TAILQ_EMPTY(&m->md.pv_list)) vm_page_aflag_clear(m, PGA_WRITEABLE); + pc->pc_map[field] |= 1UL << bit; pmap_unuse_pt(pmap, va, &free); - freemask |= 1UL << bit; freed++; } - pc->pc_map[field] |= freemask; } if (freed == 0) { TAILQ_INSERT_TAIL(&newtail, pc, pc_lru); continue; } + /* Every freed mapping is for a 4 KB page. */ pmap->pm_stats.resident_count -= freed; PV_STAT(pv_entry_frees += freed); PV_STAT(pv_entry_spare += freed); @@ -2109,13 +2108,21 @@ free_pv_entry(pmap_t pmap, pv_entry_t pv field = idx / 32; bit = idx % 32; pc->pc_map[field] |= 1ul << bit; - /* move to head of list */ - TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); for (idx = 0; idx < _NPCM; idx++) if (pc->pc_map[idx] != pc_freemask[idx]) { - TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list); + /* + * 98% of the time, pc is already at the head of the + * list. If it isn't already, move it to the head. + */ + if (__predict_false(TAILQ_FIRST(&pmap->pm_pvchunk) != + pc)) { + TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); + TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, + pc_list); + } return; } + TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); free_pv_chunk(pc); } @@ -2179,10 +2186,6 @@ retry: } TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list); - if (pc != TAILQ_LAST(&pv_chunks, pch)) { - TAILQ_REMOVE(&pv_chunks, pc, pc_lru); - TAILQ_INSERT_TAIL(&pv_chunks, pc, pc_lru); - } PV_STAT(pv_entry_spare--); return (pv); } @@ -3539,7 +3542,7 @@ pmap_remove_pages(pmap_t pmap) TAILQ_FOREACH_SAFE(pc, &pmap->pm_pvchunk, pc_list, npc) { allfree = 1; for (field = 0; field < _NPCM; field++) { - inuse = (~(pc->pc_map[field])) & pc_freemask[field]; + inuse = ~pc->pc_map[field] & pc_freemask[field]; while (inuse != 0) { bit = bsfl(inuse); bitmask = 1UL << bit;
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201207021722.q62HMcaJ083799>