From owner-dev-commits-src-main@freebsd.org Mon Dec 28 02:03:18 2020 Return-Path: Delivered-To: dev-commits-src-main@mailman.nyi.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2610:1c1:1:606c::19:1]) by mailman.nyi.freebsd.org (Postfix) with ESMTP id 19D144CDDAB; Mon, 28 Dec 2020 02:03:18 +0000 (UTC) (envelope-from git@FreeBSD.org) Received: from mxrelay.nyi.freebsd.org (mxrelay.nyi.freebsd.org [IPv6:2610:1c1:1:606c::19:3]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits) server-digest SHA256 client-signature RSA-PSS (4096 bits) client-digest SHA256) (Client CN "mxrelay.nyi.freebsd.org", Issuer "Let's Encrypt Authority X3" (verified OK)) by mx1.freebsd.org (Postfix) with ESMTPS id 4D416Z0BWVz4Xcc; Mon, 28 Dec 2020 02:03:18 +0000 (UTC) (envelope-from git@FreeBSD.org) Received: from gitrepo.freebsd.org (gitrepo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:5]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits) server-digest SHA256) (Client did not present a certificate) by mxrelay.nyi.freebsd.org (Postfix) with ESMTPS id ED28811D94; Mon, 28 Dec 2020 02:03:17 +0000 (UTC) (envelope-from git@FreeBSD.org) Received: from gitrepo.freebsd.org ([127.0.1.44]) by gitrepo.freebsd.org (8.16.1/8.16.1) with ESMTP id 0BS23HRp005319; Mon, 28 Dec 2020 02:03:17 GMT (envelope-from git@gitrepo.freebsd.org) Received: (from git@localhost) by gitrepo.freebsd.org (8.16.1/8.16.1/Submit) id 0BS23HFn005318; Mon, 28 Dec 2020 02:03:17 GMT (envelope-from git) Date: Mon, 28 Dec 2020 02:03:17 GMT Message-Id: <202012280203.0BS23HFn005318@gitrepo.freebsd.org> To: src-committers@FreeBSD.org, dev-commits-src-all@FreeBSD.org, dev-commits-src-main@FreeBSD.org From: Mateusz Guzik Subject: git: 0714f921cdcc - main - cache: save on some branching in common case mount point traversal MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit X-Git-Committer: mjg X-Git-Repository: src X-Git-Refname: refs/heads/main X-Git-Reftype: branch X-Git-Commit: 0714f921cdcc10367f258134a30eed6e5384374c Auto-Submitted: auto-generated X-BeenThere: dev-commits-src-main@freebsd.org X-Mailman-Version: 2.1.34 Precedence: list List-Id: "Commit messages for the main branch of the src repository." List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Mon, 28 Dec 2020 02:03:18 -0000 The branch main has been updated by mjg: URL: https://cgit.FreeBSD.org/src/commit/?id=0714f921cdcc10367f258134a30eed6e5384374c commit 0714f921cdcc10367f258134a30eed6e5384374c Author: Mateusz Guzik AuthorDate: 2020-12-28 01:52:20 +0000 Commit: Mateusz Guzik CommitDate: 2020-12-28 01:53:28 +0000 cache: save on some branching in common case mount point traversal --- sys/kern/vfs_cache.c | 71 ++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 64 insertions(+), 7 deletions(-) diff --git a/sys/kern/vfs_cache.c b/sys/kern/vfs_cache.c index c9bc2074d5e6..83227a9b70a8 100644 --- a/sys/kern/vfs_cache.c +++ b/sys/kern/vfs_cache.c @@ -3745,7 +3745,7 @@ cache_fpl_terminated(struct cache_fpl *fpl) _Static_assert((CACHE_FPL_SUPPORTED_CN_FLAGS & CACHE_FPL_INTERNAL_CN_FLAGS) == 0, "supported and internal flags overlap"); -static bool cache_fplookup_need_climb_mount(struct cache_fpl *fpl); +static bool cache_fplookup_is_mp(struct cache_fpl *fpl); static bool cache_fpl_islastcn(struct nameidata *ndp) @@ -4069,7 +4069,7 @@ cache_fplookup_final_modifying(struct cache_fpl *fpl) * almost never be true. */ if (__predict_false(!cache_fplookup_vnode_supported(tvp) || - cache_fplookup_need_climb_mount(fpl))) { + cache_fplookup_is_mp(fpl))) { vput(dvp); vput(tvp); return (cache_fpl_aborted(fpl)); @@ -4308,7 +4308,7 @@ cache_fplookup_noentry(struct cache_fpl *fpl) } if (__predict_false(!cache_fplookup_vnode_supported(tvp) || - cache_fplookup_need_climb_mount(fpl))) { + cache_fplookup_is_mp(fpl))) { vput(dvp); vput(tvp); return (cache_fpl_aborted(fpl)); @@ -4544,8 +4544,9 @@ cache_fplookup_climb_mount(struct cache_fpl *fpl) VNPASS(vp->v_type == VDIR || vp->v_type == VBAD, vp); mp = atomic_load_ptr(&vp->v_mountedhere); - if (mp == NULL) + if (__predict_false(mp == NULL)) { return (0); + } prev_mp = NULL; for (;;) { @@ -4587,8 +4588,64 @@ cache_fplookup_climb_mount(struct cache_fpl *fpl) return (0); } +static int __noinline +cache_fplookup_cross_mount(struct cache_fpl *fpl) +{ + struct mount *mp; + struct mount_pcpu *mpcpu; + struct vnode *vp; + seqc_t vp_seqc; + + vp = fpl->tvp; + vp_seqc = fpl->tvp_seqc; + + VNPASS(vp->v_type == VDIR || vp->v_type == VBAD, vp); + mp = atomic_load_ptr(&vp->v_mountedhere); + if (__predict_false(mp == NULL)) { + return (0); + } + + if (!vfs_op_thread_enter_crit(mp, mpcpu)) { + return (cache_fpl_partial(fpl)); + } + if (!vn_seqc_consistent(vp, vp_seqc)) { + vfs_op_thread_exit_crit(mp, mpcpu); + return (cache_fpl_partial(fpl)); + } + if (!cache_fplookup_mp_supported(mp)) { + vfs_op_thread_exit_crit(mp, mpcpu); + return (cache_fpl_partial(fpl)); + } + vp = atomic_load_ptr(&mp->mnt_rootvnode); + if (__predict_false(vp == NULL || VN_IS_DOOMED(vp))) { + vfs_op_thread_exit_crit(mp, mpcpu); + return (cache_fpl_partial(fpl)); + } + vp_seqc = vn_seqc_read_any(vp); + vfs_op_thread_exit_crit(mp, mpcpu); + if (seqc_in_modify(vp_seqc)) { + return (cache_fpl_partial(fpl)); + } + mp = atomic_load_ptr(&vp->v_mountedhere); + if (__predict_false(mp != NULL)) { + /* + * There are possibly more mount points stack on top. + * Normally this does not happen so for simplicity just start + * over. + */ + return (cache_fplookup_climb_mount(fpl)); + } + + fpl->tvp = vp; + fpl->tvp_seqc = vp_seqc; + return (0); +} + +/* + * Check if a vnode mounted on. + */ static bool -cache_fplookup_need_climb_mount(struct cache_fpl *fpl) +cache_fplookup_is_mp(struct cache_fpl *fpl) { struct mount *mp; struct vnode *vp; @@ -4834,8 +4891,8 @@ cache_fplookup_impl(struct vnode *dvp, struct cache_fpl *fpl) VNPASS(!seqc_in_modify(fpl->tvp_seqc), fpl->tvp); - if (cache_fplookup_need_climb_mount(fpl)) { - error = cache_fplookup_climb_mount(fpl); + if (cache_fplookup_is_mp(fpl)) { + error = cache_fplookup_cross_mount(fpl); if (__predict_false(error != 0)) { break; }