From owner-svn-src-head@freebsd.org Wed Aug 5 09:25:59 2020 Return-Path: Delivered-To: svn-src-head@mailman.nyi.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2610:1c1:1:606c::19:1]) by mailman.nyi.freebsd.org (Postfix) with ESMTP id F06123AFA8B; Wed, 5 Aug 2020 09:25:59 +0000 (UTC) (envelope-from mjg@FreeBSD.org) Received: from mxrelay.nyi.freebsd.org (mxrelay.nyi.freebsd.org [IPv6:2610:1c1:1:606c::19:3]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits) server-digest SHA256 client-signature RSA-PSS (4096 bits) client-digest SHA256) (Client CN "mxrelay.nyi.freebsd.org", Issuer "Let's Encrypt Authority X3" (verified OK)) by mx1.freebsd.org (Postfix) with ESMTPS id 4BM5pH66bzz4Znf; Wed, 5 Aug 2020 09:25:59 +0000 (UTC) (envelope-from mjg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mxrelay.nyi.freebsd.org (Postfix) with ESMTPS id B4F1413D4F; Wed, 5 Aug 2020 09:25:59 +0000 (UTC) (envelope-from mjg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id 0759Pxeo011022; Wed, 5 Aug 2020 09:25:59 GMT (envelope-from mjg@FreeBSD.org) Received: (from mjg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id 0759Px4U011021; Wed, 5 Aug 2020 09:25:59 GMT (envelope-from mjg@FreeBSD.org) Message-Id: <202008050925.0759Px4U011021@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: mjg set sender to mjg@FreeBSD.org using -f From: Mateusz Guzik Date: Wed, 5 Aug 2020 09:25:59 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r363892 - head/sys/kern X-SVN-Group: head X-SVN-Commit-Author: mjg X-SVN-Commit-Paths: head/sys/kern X-SVN-Commit-Revision: 363892 X-SVN-Commit-Repository: base MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-head@freebsd.org X-Mailman-Version: 2.1.33 Precedence: list List-Id: SVN commit messages for the src tree for head/-current List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 05 Aug 2020 09:26:00 -0000 Author: mjg Date: Wed Aug 5 09:25:59 2020 New Revision: 363892 URL: https://svnweb.freebsd.org/changeset/base/363892 Log: cache: convert the hash from LIST to SLIST This reduces struct namecache by sizeof(void *). Negative side is that we have to find the previous element (if any) when removing an entry, but since we normally don't expect collisions it should be fine. Note this adds cache_get_hash calls which can be eliminated. Modified: head/sys/kern/vfs_cache.c Modified: head/sys/kern/vfs_cache.c ============================================================================== --- head/sys/kern/vfs_cache.c Wed Aug 5 09:24:38 2020 (r363891) +++ head/sys/kern/vfs_cache.c Wed Aug 5 09:25:59 2020 (r363892) @@ -124,7 +124,7 @@ _Static_assert(sizeof(struct negstate) <= sizeof(struc struct namecache { LIST_ENTRY(namecache) nc_src; /* source vnode list */ TAILQ_ENTRY(namecache) nc_dst; /* destination vnode list */ - CK_LIST_ENTRY(namecache) nc_hash;/* hash chain */ + CK_SLIST_ENTRY(namecache) nc_hash;/* hash chain */ struct vnode *nc_dvp; /* vnode of parent of name */ union { struct vnode *nu_vp; /* vnode the name refers to */ @@ -264,7 +264,7 @@ VFS_SMR_DECLARE; */ #define NCHHASH(hash) \ (&nchashtbl[(hash) & nchash]) -static __read_mostly CK_LIST_HEAD(nchashhead, namecache) *nchashtbl;/* Hash Table */ +static __read_mostly CK_SLIST_HEAD(nchashhead, namecache) *nchashtbl;/* Hash Table */ static u_long __read_mostly nchash; /* size of hash table */ SYSCTL_ULONG(_debug, OID_AUTO, nchash, CTLFLAG_RD, &nchash, 0, "Size of namecache hash table"); @@ -520,6 +520,15 @@ cache_get_hash(char *name, u_char len, struct vnode *d return (fnv_32_buf(name, len, dvp->v_nchash)); } +static inline struct nchashhead * +NCP2BUCKET(struct namecache *ncp) +{ + uint32_t hash; + + hash = cache_get_hash(ncp->nc_name, ncp->nc_nlen, ncp->nc_dvp); + return (NCHHASH(hash)); +} + static inline struct rwlock * NCP2BUCKETLOCK(struct namecache *ncp) { @@ -687,7 +696,7 @@ retry: } /* Scan hash tables counting entries */ for (ncpp = nchashtbl, i = 0; i < n_nchash; ncpp++, i++) - CK_LIST_FOREACH(ncp, ncpp, nc_hash) + CK_SLIST_FOREACH(ncp, ncpp, nc_hash) cntbuf[i]++; cache_unlock_all_buckets(); for (error = 0, i = 0; i < n_nchash; i++) @@ -720,7 +729,7 @@ sysctl_debug_hashstat_nchash(SYSCTL_HANDLER_ARGS) /* Scan hash tables for applicable entries */ for (ncpp = nchashtbl; n_nchash > 0; n_nchash--, ncpp++) { count = 0; - CK_LIST_FOREACH(ncp, ncpp, nc_hash) { + CK_SLIST_FOREACH(ncp, ncpp, nc_hash) { count++; } if (count) @@ -952,6 +961,7 @@ cache_negative_zap_one(void) static void cache_zap_locked(struct namecache *ncp) { + struct nchashhead *ncpp; if (!(ncp->nc_flag & NCF_NEGATIVE)) cache_assert_vnode_locked(ncp->nc_vp); @@ -963,7 +973,8 @@ cache_zap_locked(struct namecache *ncp) cache_ncp_invalidate(ncp); - CK_LIST_REMOVE(ncp, nc_hash); + ncpp = NCP2BUCKET(ncp); + CK_SLIST_REMOVE(ncpp, ncp, namecache, nc_hash); if (!(ncp->nc_flag & NCF_NEGATIVE)) { SDT_PROBE3(vfs, namecache, zap, done, ncp->nc_dvp, ncp->nc_name, ncp->nc_vp); @@ -1122,7 +1133,7 @@ cache_zap_unlocked_bucket(struct namecache *ncp, struc cache_sort_vnodes(&dvlp, &vlp); cache_lock_vnodes(dvlp, vlp); rw_wlock(blp); - CK_LIST_FOREACH(rncp, (NCHHASH(hash)), nc_hash) { + CK_SLIST_FOREACH(rncp, (NCHHASH(hash)), nc_hash) { if (rncp == ncp && rncp->nc_dvp == dvp && rncp->nc_nlen == cnp->cn_namelen && !bcmp(rncp->nc_name, cnp->cn_nameptr, rncp->nc_nlen)) @@ -1336,12 +1347,12 @@ retry_dotdot: hash = cache_get_hash(cnp->cn_nameptr, cnp->cn_namelen, dvp); blp = HASH2BUCKETLOCK(hash); retry: - if (CK_LIST_EMPTY(NCHHASH(hash))) + if (CK_SLIST_EMPTY(NCHHASH(hash))) goto out_no_entry; rw_wlock(blp); - CK_LIST_FOREACH(ncp, (NCHHASH(hash)), nc_hash) { + CK_SLIST_FOREACH(ncp, (NCHHASH(hash)), nc_hash) { if (ncp->nc_dvp == dvp && ncp->nc_nlen == cnp->cn_namelen && !bcmp(ncp->nc_name, cnp->cn_nameptr, ncp->nc_nlen)) break; @@ -1485,7 +1496,7 @@ retry_hashed: rw_rlock(blp); } - CK_LIST_FOREACH(ncp, (NCHHASH(hash)), nc_hash) { + CK_SLIST_FOREACH(ncp, (NCHHASH(hash)), nc_hash) { if (ncp->nc_dvp == dvp && ncp->nc_nlen == cnp->cn_namelen && !bcmp(ncp->nc_name, cnp->cn_nameptr, ncp->nc_nlen)) break; @@ -1932,7 +1943,7 @@ cache_enter_time(struct vnode *dvp, struct vnode *vp, * the same path name. */ ncpp = NCHHASH(hash); - CK_LIST_FOREACH(n2, ncpp, nc_hash) { + CK_SLIST_FOREACH(n2, ncpp, nc_hash) { if (n2->nc_dvp == dvp && n2->nc_nlen == cnp->cn_namelen && !bcmp(n2->nc_name, cnp->cn_nameptr, n2->nc_nlen)) { @@ -2021,7 +2032,7 @@ cache_enter_time(struct vnode *dvp, struct vnode *vp, * Insert the new namecache entry into the appropriate chain * within the cache entries table. */ - CK_LIST_INSERT_HEAD(ncpp, ncp, nc_hash); + CK_SLIST_INSERT_HEAD(ncpp, ncp, nc_hash); atomic_thread_fence_rel(); /* @@ -2053,6 +2064,28 @@ cache_roundup_2(u_int val) return (res); } +static struct nchashhead * +nchinittbl(u_long elements, u_long *hashmask) +{ + struct nchashhead *hashtbl; + u_long hashsize, i; + + hashsize = cache_roundup_2(desiredvnodes * 2) / 2; + + hashtbl = malloc((u_long)hashsize * sizeof(*hashtbl), M_VFSCACHE, M_WAITOK); + for (i = 0; i < hashsize; i++) + CK_SLIST_INIT(&hashtbl[i]); + *hashmask = hashsize - 1; + return (hashtbl); +} + +static void +ncfreetbl(struct nchashhead *hashtbl) +{ + + free(hashtbl, M_VFSCACHE); +} + /* * Name cache initialization, from vfs_init() when we are booting */ @@ -2084,7 +2117,7 @@ nchinit(void *dummy __unused) VFS_SMR_ZONE_SET(cache_zone_large_ts); ncsize = desiredvnodes * ncsizefactor; - nchashtbl = hashinit(desiredvnodes * 2, M_VFSCACHE, &nchash); + nchashtbl = nchinittbl(desiredvnodes * 2, &nchash); ncbuckethash = cache_roundup_2(mp_ncpus * mp_ncpus) - 1; if (ncbuckethash < 7) /* arbitrarily chosen to avoid having one lock */ ncbuckethash = 7; @@ -2139,10 +2172,10 @@ cache_changesize(u_long newmaxvnodes) if (newmaxvnodes < numbucketlocks) newmaxvnodes = numbucketlocks; - new_nchashtbl = hashinit(newmaxvnodes, M_VFSCACHE, &new_nchash); + new_nchashtbl = nchinittbl(newmaxvnodes, &new_nchash); /* If same hash table size, nothing to do */ if (nchash == new_nchash) { - free(new_nchashtbl, M_VFSCACHE); + ncfreetbl(new_nchashtbl); return; } /* @@ -2157,17 +2190,17 @@ cache_changesize(u_long newmaxvnodes) nchashtbl = new_nchashtbl; nchash = new_nchash; for (i = 0; i <= old_nchash; i++) { - while ((ncp = CK_LIST_FIRST(&old_nchashtbl[i])) != NULL) { + while ((ncp = CK_SLIST_FIRST(&old_nchashtbl[i])) != NULL) { hash = cache_get_hash(ncp->nc_name, ncp->nc_nlen, ncp->nc_dvp); - CK_LIST_REMOVE(ncp, nc_hash); - CK_LIST_INSERT_HEAD(NCHHASH(hash), ncp, nc_hash); + CK_SLIST_REMOVE(&old_nchashtbl[i], ncp, namecache, nc_hash); + CK_SLIST_INSERT_HEAD(NCHHASH(hash), ncp, nc_hash); } } ncsize = newncsize; cache_unlock_all_buckets(); cache_unlock_all_vnodes(); - free(old_nchashtbl, M_VFSCACHE); + ncfreetbl(old_nchashtbl); } /* @@ -2317,7 +2350,7 @@ cache_purgevfs(struct mount *mp, bool force) for (j = i; j < n_nchash; j += numbucketlocks) { retry: bucket = &nchashtbl[j]; - CK_LIST_FOREACH_SAFE(ncp, bucket, nc_hash, nnp) { + CK_SLIST_FOREACH_SAFE(ncp, bucket, nc_hash, nnp) { cache_assert_bucket_locked(ncp, RA_WLOCKED); if (ncp->nc_dvp->v_mount != mp) continue; @@ -3233,7 +3266,7 @@ cache_fplookup_negative_promote(struct cache_fpl *fpl, * In particular at this point there can be a new ncp which matches the * search but hashes to a different neglist. */ - CK_LIST_FOREACH(ncp, (NCHHASH(hash)), nc_hash) { + CK_SLIST_FOREACH(ncp, (NCHHASH(hash)), nc_hash) { if (ncp == oncp) break; } @@ -3583,7 +3616,7 @@ cache_fplookup_next(struct cache_fpl *fpl) hash = cache_get_hash(cnp->cn_nameptr, cnp->cn_namelen, dvp); - CK_LIST_FOREACH(ncp, (NCHHASH(hash)), nc_hash) { + CK_SLIST_FOREACH(ncp, (NCHHASH(hash)), nc_hash) { if (ncp->nc_dvp == dvp && ncp->nc_nlen == cnp->cn_namelen && !bcmp(ncp->nc_name, cnp->cn_nameptr, ncp->nc_nlen)) break;