Date: Mon, 4 Jun 2012 03:51:09 +0000 (UTC) From: Alan Cox <alc@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r236534 - in head/sys: amd64/amd64 i386/i386 i386/xen Message-ID: <201206040351.q543p9YX002912@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: alc Date: Mon Jun 4 03:51:08 2012 New Revision: 236534 URL: http://svn.freebsd.org/changeset/base/236534 Log: Various small changes to PV entry management: Constify pc_freemask[]. pmap_pv_reclaim() Eliminate "freemask" because it was a pessimization. Add a comment about the resident count adjustment. free_pv_entry() [i386 only] Merge an optimization from amd64 (r233954). get_pv_entry() Eliminate the move to tail of the pv_chunk on the global pv_chunks list. (The right strategy needs more thought. Moreover, there were unintended differences between the amd64 and i386 implementation.) pmap_remove_pages() Eliminate unnecessary ()'s. Modified: head/sys/amd64/amd64/pmap.c head/sys/i386/i386/pmap.c head/sys/i386/xen/pmap.c Modified: head/sys/amd64/amd64/pmap.c ============================================================================== --- head/sys/amd64/amd64/pmap.c Mon Jun 4 02:25:11 2012 (r236533) +++ head/sys/amd64/amd64/pmap.c Mon Jun 4 03:51:08 2012 (r236534) @@ -2021,7 +2021,7 @@ pv_to_chunk(pv_entry_t pv) #define PC_FREE1 0xfffffffffffffffful #define PC_FREE2 0x000000fffffffffful -static uint64_t pc_freemask[_NPCM] = { PC_FREE0, PC_FREE1, PC_FREE2 }; +static const uint64_t pc_freemask[_NPCM] = { PC_FREE0, PC_FREE1, PC_FREE2 }; SYSCTL_LONG(_vm_pmap, OID_AUTO, pv_entry_count, CTLFLAG_RD, &pv_entry_count, 0, "Current number of pv entries"); @@ -2070,7 +2070,7 @@ pmap_pv_reclaim(pmap_t locked_pmap) pv_entry_t pv; vm_offset_t va; vm_page_t free, m, m_pc; - uint64_t inuse, freemask; + uint64_t inuse; int bit, field, freed; rw_assert(&pvh_global_lock, RA_WLOCKED); @@ -2102,7 +2102,6 @@ pmap_pv_reclaim(pmap_t locked_pmap) */ freed = 0; for (field = 0; field < _NPCM; field++) { - freemask = 0; for (inuse = ~pc->pc_map[field] & pc_freemask[field]; inuse != 0; inuse &= ~(1UL << bit)) { bit = bsfq(inuse); @@ -2131,16 +2130,16 @@ pmap_pv_reclaim(pmap_t locked_pmap) PGA_WRITEABLE); } } + pc->pc_map[field] |= 1UL << bit; pmap_unuse_pt(pmap, va, *pde, &free); - freemask |= 1UL << bit; freed++; } - pc->pc_map[field] |= freemask; } if (freed == 0) { TAILQ_INSERT_TAIL(&newtail, pc, pc_lru); continue; } + /* Every freed mapping is for a 4 KB page. */ pmap_resident_count_dec(pmap, freed); PV_STAT(pv_entry_frees += freed); PV_STAT(pv_entry_spare += freed); @@ -2261,10 +2260,6 @@ retry: TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list); } - if (pc != TAILQ_LAST(&pv_chunks, pch)) { - TAILQ_REMOVE(&pv_chunks, pc, pc_lru); - TAILQ_INSERT_TAIL(&pv_chunks, pc, pc_lru); - } pv_entry_count++; PV_STAT(pv_entry_spare--); return (pv); @@ -4137,7 +4132,7 @@ pmap_remove_pages(pmap_t pmap) TAILQ_FOREACH_SAFE(pc, &pmap->pm_pvchunk, pc_list, npc) { allfree = 1; for (field = 0; field < _NPCM; field++) { - inuse = (~(pc->pc_map[field])) & pc_freemask[field]; + inuse = ~pc->pc_map[field] & pc_freemask[field]; while (inuse != 0) { bit = bsfq(inuse); bitmask = 1UL << bit; Modified: head/sys/i386/i386/pmap.c ============================================================================== --- head/sys/i386/i386/pmap.c Mon Jun 4 02:25:11 2012 (r236533) +++ head/sys/i386/i386/pmap.c Mon Jun 4 03:51:08 2012 (r236534) @@ -2177,7 +2177,7 @@ pv_to_chunk(pv_entry_t pv) #define PC_FREE0_9 0xfffffffful /* Free values for index 0 through 9 */ #define PC_FREE10 0x0000fffful /* Free values for index 10 */ -static uint32_t pc_freemask[_NPCM] = { +static const uint32_t pc_freemask[_NPCM] = { PC_FREE0_9, PC_FREE0_9, PC_FREE0_9, PC_FREE0_9, PC_FREE0_9, PC_FREE0_9, PC_FREE0_9, PC_FREE0_9, PC_FREE0_9, @@ -2227,7 +2227,7 @@ pmap_pv_reclaim(pmap_t locked_pmap) pv_entry_t pv; vm_offset_t va; vm_page_t free, m, m_pc; - uint32_t inuse, freemask; + uint32_t inuse; int bit, field, freed; PMAP_LOCK_ASSERT(locked_pmap, MA_OWNED); @@ -2260,7 +2260,6 @@ pmap_pv_reclaim(pmap_t locked_pmap) */ freed = 0; for (field = 0; field < _NPCM; field++) { - freemask = 0; for (inuse = ~pc->pc_map[field] & pc_freemask[field]; inuse != 0; inuse &= ~(1UL << bit)) { bit = bsfl(inuse); @@ -2289,16 +2288,16 @@ pmap_pv_reclaim(pmap_t locked_pmap) PGA_WRITEABLE); } } + pc->pc_map[field] |= 1UL << bit; pmap_unuse_pt(pmap, va, &free); - freemask |= 1UL << bit; freed++; } - pc->pc_map[field] |= freemask; } if (freed == 0) { TAILQ_INSERT_TAIL(&newtail, pc, pc_lru); continue; } + /* Every freed mapping is for a 4 KB page. */ pmap->pm_stats.resident_count -= freed; PV_STAT(pv_entry_frees += freed); PV_STAT(pv_entry_spare += freed); @@ -2367,13 +2366,21 @@ free_pv_entry(pmap_t pmap, pv_entry_t pv field = idx / 32; bit = idx % 32; pc->pc_map[field] |= 1ul << bit; - /* move to head of list */ - TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); for (idx = 0; idx < _NPCM; idx++) if (pc->pc_map[idx] != pc_freemask[idx]) { - TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list); + /* + * 98% of the time, pc is already at the head of the + * list. If it isn't already, move it to the head. + */ + if (__predict_false(TAILQ_FIRST(&pmap->pm_pvchunk) != + pc)) { + TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); + TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, + pc_list); + } return; } + TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); free_pv_chunk(pc); } @@ -2437,10 +2444,6 @@ retry: } TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list); - if (pc != TAILQ_LAST(&pv_chunks, pch)) { - TAILQ_REMOVE(&pv_chunks, pc, pc_lru); - TAILQ_INSERT_TAIL(&pv_chunks, pc, pc_lru); - } PV_STAT(pv_entry_spare--); return (pv); } @@ -4371,7 +4374,7 @@ pmap_remove_pages(pmap_t pmap) TAILQ_FOREACH_SAFE(pc, &pmap->pm_pvchunk, pc_list, npc) { allfree = 1; for (field = 0; field < _NPCM; field++) { - inuse = (~(pc->pc_map[field])) & pc_freemask[field]; + inuse = ~pc->pc_map[field] & pc_freemask[field]; while (inuse != 0) { bit = bsfl(inuse); bitmask = 1UL << bit; Modified: head/sys/i386/xen/pmap.c ============================================================================== --- head/sys/i386/xen/pmap.c Mon Jun 4 02:25:11 2012 (r236533) +++ head/sys/i386/xen/pmap.c Mon Jun 4 03:51:08 2012 (r236534) @@ -1929,7 +1929,7 @@ pv_to_chunk(pv_entry_t pv) #define PC_FREE0_9 0xfffffffful /* Free values for index 0 through 9 */ #define PC_FREE10 0x0000fffful /* Free values for index 10 */ -static uint32_t pc_freemask[_NPCM] = { +static const uint32_t pc_freemask[_NPCM] = { PC_FREE0_9, PC_FREE0_9, PC_FREE0_9, PC_FREE0_9, PC_FREE0_9, PC_FREE0_9, PC_FREE0_9, PC_FREE0_9, PC_FREE0_9, @@ -1977,7 +1977,7 @@ pmap_pv_reclaim(pmap_t locked_pmap) pv_entry_t pv; vm_offset_t va; vm_page_t free, m, m_pc; - uint32_t inuse, freemask; + uint32_t inuse; int bit, field, freed; PMAP_LOCK_ASSERT(locked_pmap, MA_OWNED); @@ -2010,7 +2010,6 @@ pmap_pv_reclaim(pmap_t locked_pmap) */ freed = 0; for (field = 0; field < _NPCM; field++) { - freemask = 0; for (inuse = ~pc->pc_map[field] & pc_freemask[field]; inuse != 0; inuse &= ~(1UL << bit)) { bit = bsfl(inuse); @@ -2030,16 +2029,16 @@ pmap_pv_reclaim(pmap_t locked_pmap) TAILQ_REMOVE(&m->md.pv_list, pv, pv_list); if (TAILQ_EMPTY(&m->md.pv_list)) vm_page_aflag_clear(m, PGA_WRITEABLE); + pc->pc_map[field] |= 1UL << bit; pmap_unuse_pt(pmap, va, &free); - freemask |= 1UL << bit; freed++; } - pc->pc_map[field] |= freemask; } if (freed == 0) { TAILQ_INSERT_TAIL(&newtail, pc, pc_lru); continue; } + /* Every freed mapping is for a 4 KB page. */ pmap->pm_stats.resident_count -= freed; PV_STAT(pv_entry_frees += freed); PV_STAT(pv_entry_spare += freed); @@ -2108,13 +2107,21 @@ free_pv_entry(pmap_t pmap, pv_entry_t pv field = idx / 32; bit = idx % 32; pc->pc_map[field] |= 1ul << bit; - /* move to head of list */ - TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); for (idx = 0; idx < _NPCM; idx++) if (pc->pc_map[idx] != pc_freemask[idx]) { - TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list); + /* + * 98% of the time, pc is already at the head of the + * list. If it isn't already, move it to the head. + */ + if (__predict_false(TAILQ_FIRST(&pmap->pm_pvchunk) != + pc)) { + TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); + TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, + pc_list); + } return; } + TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); free_pv_chunk(pc); } @@ -2178,10 +2185,6 @@ retry: } TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list); - if (pc != TAILQ_LAST(&pv_chunks, pch)) { - TAILQ_REMOVE(&pv_chunks, pc, pc_lru); - TAILQ_INSERT_TAIL(&pv_chunks, pc, pc_lru); - } PV_STAT(pv_entry_spare--); return (pv); } @@ -3538,7 +3541,7 @@ pmap_remove_pages(pmap_t pmap) TAILQ_FOREACH_SAFE(pc, &pmap->pm_pvchunk, pc_list, npc) { allfree = 1; for (field = 0; field < _NPCM; field++) { - inuse = (~(pc->pc_map[field])) & pc_freemask[field]; + inuse = ~pc->pc_map[field] & pc_freemask[field]; while (inuse != 0) { bit = bsfl(inuse); bitmask = 1UL << bit;
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201206040351.q543p9YX002912>