From owner-svn-src-head@freebsd.org Fri Oct 16 00:56:14 2020 Return-Path: Delivered-To: svn-src-head@mailman.nyi.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2610:1c1:1:606c::19:1]) by mailman.nyi.freebsd.org (Postfix) with ESMTP id C712544B248; Fri, 16 Oct 2020 00:56:14 +0000 (UTC) (envelope-from mjg@FreeBSD.org) Received: from mxrelay.nyi.freebsd.org (mxrelay.nyi.freebsd.org [IPv6:2610:1c1:1:606c::19:3]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits) server-digest SHA256 client-signature RSA-PSS (4096 bits) client-digest SHA256) (Client CN "mxrelay.nyi.freebsd.org", Issuer "Let's Encrypt Authority X3" (verified OK)) by mx1.freebsd.org (Postfix) with ESMTPS id 4CC74t4Zqlz4FDk; Fri, 16 Oct 2020 00:56:14 +0000 (UTC) (envelope-from mjg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mxrelay.nyi.freebsd.org (Postfix) with ESMTPS id 814DD84EC; Fri, 16 Oct 2020 00:56:14 +0000 (UTC) (envelope-from mjg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id 09G0uEmh086465; Fri, 16 Oct 2020 00:56:14 GMT (envelope-from mjg@FreeBSD.org) Received: (from mjg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id 09G0uEag086464; Fri, 16 Oct 2020 00:56:14 GMT (envelope-from mjg@FreeBSD.org) Message-Id: <202010160056.09G0uEag086464@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: mjg set sender to mjg@FreeBSD.org using -f From: Mateusz Guzik Date: Fri, 16 Oct 2020 00:56:14 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r366743 - head/sys/kern X-SVN-Group: head X-SVN-Commit-Author: mjg X-SVN-Commit-Paths: head/sys/kern X-SVN-Commit-Revision: 366743 X-SVN-Commit-Repository: base MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-head@freebsd.org X-Mailman-Version: 2.1.33 Precedence: list List-Id: SVN commit messages for the src tree for head/-current List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 16 Oct 2020 00:56:14 -0000 Author: mjg Date: Fri Oct 16 00:56:13 2020 New Revision: 366743 URL: https://svnweb.freebsd.org/changeset/base/366743 Log: cache: support negative entry promotion in slowpath smr Modified: head/sys/kern/vfs_cache.c Modified: head/sys/kern/vfs_cache.c ============================================================================== --- head/sys/kern/vfs_cache.c Fri Oct 16 00:55:57 2020 (r366742) +++ head/sys/kern/vfs_cache.c Fri Oct 16 00:56:13 2020 (r366743) @@ -806,6 +806,9 @@ cache_negative_init(struct namecache *ncp) ns->neg_flag = 0; } +/* + * Move a negative entry to the hot list. + */ static void cache_negative_promote(struct namecache *ncp) { @@ -823,6 +826,87 @@ cache_negative_promote(struct namecache *ncp) } } +/* + * Move a negative entry to the hot list if it matches the lookup. + * + * We have to take locks, but they may be contended and in the worst + * case we may need to go off CPU. We don't want to spin within the + * smr section and we can't block with it. Exiting the section means + * the found entry could have been evicted. We are going to look it + * up again. + */ +static bool +cache_negative_promote_cond(struct vnode *dvp, struct componentname *cnp, + struct namecache *oncp, uint32_t hash) +{ + struct namecache *ncp; + struct neglist *nl; + u_char nc_flag; + + nl = NCP2NEGLIST(oncp); + + mtx_lock(&nl->nl_lock); + /* + * For hash iteration. + */ + vfs_smr_enter(); + + /* + * Avoid all surprises by only succeeding if we got the same entry and + * bailing completely otherwise. + * XXX There are no provisions to keep the vnode around, meaning we may + * end up promoting a negative entry for a *new* vnode and returning + * ENOENT on its account. This is the error we want to return anyway + * and promotion is harmless. + * + * In particular at this point there can be a new ncp which matches the + * search but hashes to a different neglist. + */ + CK_SLIST_FOREACH(ncp, (NCHHASH(hash)), nc_hash) { + if (ncp == oncp) + break; + } + + /* + * No match to begin with. + */ + if (__predict_false(ncp == NULL)) { + goto out_abort; + } + + /* + * The newly found entry may be something different... + */ + if (!(ncp->nc_dvp == dvp && ncp->nc_nlen == cnp->cn_namelen && + !bcmp(ncp->nc_name, cnp->cn_nameptr, ncp->nc_nlen))) { + goto out_abort; + } + + /* + * ... and not even negative. + */ + nc_flag = atomic_load_char(&ncp->nc_flag); + if ((nc_flag & NCF_NEGATIVE) == 0) { + goto out_abort; + } + + if (__predict_false(!cache_ncp_canuse(ncp))) { + goto out_abort; + } + + cache_negative_promote(ncp); + + SDT_PROBE2(vfs, namecache, lookup, hit__negative, dvp, ncp->nc_name); + counter_u64_add(numneghits, 1); + vfs_smr_exit(); + mtx_unlock(&nl->nl_lock); + return (true); +out_abort: + vfs_smr_exit(); + mtx_unlock(&nl->nl_lock); + return (false); +} + static void cache_negative_hit(struct namecache *ncp) { @@ -1455,7 +1539,7 @@ cache_lookup(struct vnode *dvp, struct vnode **vpp, st uint32_t hash; enum vgetstate vs; int error; - bool whiteout; + bool whiteout, neg_hot; u_short nc_flag; MPASS((tsp == NULL && ticksp == NULL) || (tsp != NULL && ticksp != NULL)); @@ -1532,21 +1616,23 @@ negative_success: } } - SDT_PROBE2(vfs, namecache, lookup, hit__negative, dvp, ncp->nc_name); cache_out_ts(ncp, tsp, ticksp); - counter_u64_add(numneghits, 1); whiteout = (ncp->nc_flag & NCF_WHITE); - /* - * TODO: We need to take locks to promote an entry. Code doing it - * in SMR lookup can be modified to be shared. - */ ns = NCP2NEGSTATE(ncp); - if ((ns->neg_flag & NEG_HOT) == 0 || - !cache_ncp_canuse(ncp)) { + neg_hot = ((ns->neg_flag & NEG_HOT) != 0); + if (__predict_false(!cache_ncp_canuse(ncp))) { vfs_smr_exit(); goto out_fallback; } - vfs_smr_exit(); + if (neg_hot) { + vfs_smr_exit(); + if (!cache_negative_promote_cond(dvp, cnp, ncp, hash)) + goto out_fallback; + } else { + SDT_PROBE2(vfs, namecache, lookup, hit__negative, dvp, ncp->nc_name); + counter_u64_add(numneghits, 1); + vfs_smr_exit(); + } if (whiteout) cnp->cn_flags |= ISWHITEOUT; return (ENOENT); @@ -3373,90 +3459,21 @@ cache_fplookup_vnode_supported(struct vnode *vp) return (vp->v_type != VLNK); } -/* - * Move a negative entry to the hot list. - * - * We have to take locks, but they may be contended and in the worst - * case we may need to go off CPU. We don't want to spin within the - * smr section and we can't block with it. Instead we are going to - * look up the entry again. - */ static int __noinline cache_fplookup_negative_promote(struct cache_fpl *fpl, struct namecache *oncp, uint32_t hash) { struct componentname *cnp; - struct namecache *ncp; - struct neglist *nl; struct vnode *dvp; - u_char nc_flag; cnp = fpl->cnp; dvp = fpl->dvp; - nl = NCP2NEGLIST(oncp); cache_fpl_smr_exit(fpl); - - mtx_lock(&nl->nl_lock); - /* - * For hash iteration. - */ - cache_fpl_smr_enter(fpl); - - /* - * Avoid all surprises by only succeeding if we got the same entry and - * bailing completely otherwise. - * XXX There are no provisions to keep the vnode around, meaning we may - * end up promoting a negative entry for a *new* vnode and returning - * ENOENT on its account. This is the error we want to return anyway - * and promotion is harmless. - * - * In particular at this point there can be a new ncp which matches the - * search but hashes to a different neglist. - */ - CK_SLIST_FOREACH(ncp, (NCHHASH(hash)), nc_hash) { - if (ncp == oncp) - break; - } - - /* - * No match to begin with. - */ - if (__predict_false(ncp == NULL)) { - goto out_abort; - } - - /* - * The newly found entry may be something different... - */ - if (!(ncp->nc_dvp == dvp && ncp->nc_nlen == cnp->cn_namelen && - !bcmp(ncp->nc_name, cnp->cn_nameptr, ncp->nc_nlen))) { - goto out_abort; - } - - /* - * ... and not even negative. - */ - nc_flag = atomic_load_char(&ncp->nc_flag); - if ((nc_flag & NCF_NEGATIVE) == 0) { - goto out_abort; - } - - if (__predict_false(!cache_ncp_canuse(ncp))) { - goto out_abort; - } - - cache_negative_promote(ncp); - - SDT_PROBE2(vfs, namecache, lookup, hit__negative, dvp, ncp->nc_name); - counter_u64_add(numneghits, 1); - cache_fpl_smr_exit(fpl); - mtx_unlock(&nl->nl_lock); - return (cache_fpl_handled(fpl, ENOENT)); -out_abort: - cache_fpl_smr_exit(fpl); - mtx_unlock(&nl->nl_lock); - return (cache_fpl_aborted(fpl)); + if (cache_negative_promote_cond(dvp, cnp, oncp, hash)) + return (cache_fpl_handled(fpl, ENOENT)); + else + return (cache_fpl_aborted(fpl)); } /*