Date: Mon, 9 Nov 2020 23:02:13 +0000 (UTC) From: Mateusz Guzik <mjg@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r367535 - in head/sys: kern sys Message-ID: <202011092302.0A9N2D63015034@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: mjg Date: Mon Nov 9 23:02:13 2020 New Revision: 367535 URL: https://svnweb.freebsd.org/changeset/base/367535 Log: vfs: group mount per-cpu vars into one struct While here move frequently read stuff into the same cacheline. This shrinks struct mount by 64 bytes. Tested by: pho Modified: head/sys/kern/vfs_cache.c head/sys/kern/vfs_default.c head/sys/kern/vfs_mount.c head/sys/kern/vfs_subr.c head/sys/kern/vfs_vnops.c head/sys/sys/mount.h Modified: head/sys/kern/vfs_cache.c ============================================================================== --- head/sys/kern/vfs_cache.c Mon Nov 9 23:00:29 2020 (r367534) +++ head/sys/kern/vfs_cache.c Mon Nov 9 23:02:13 2020 (r367535) @@ -4249,6 +4249,7 @@ static int __noinline cache_fplookup_climb_mount(struct cache_fpl *fpl) { struct mount *mp, *prev_mp; + struct mount_pcpu *mpcpu, *prev_mpcpu; struct vnode *vp; seqc_t vp_seqc; @@ -4262,38 +4263,39 @@ cache_fplookup_climb_mount(struct cache_fpl *fpl) prev_mp = NULL; for (;;) { - if (!vfs_op_thread_enter_crit(mp)) { + if (!vfs_op_thread_enter_crit(mp, mpcpu)) { if (prev_mp != NULL) - vfs_op_thread_exit_crit(prev_mp); + vfs_op_thread_exit_crit(prev_mp, prev_mpcpu); return (cache_fpl_partial(fpl)); } if (prev_mp != NULL) - vfs_op_thread_exit_crit(prev_mp); + vfs_op_thread_exit_crit(prev_mp, prev_mpcpu); if (!vn_seqc_consistent(vp, vp_seqc)) { - vfs_op_thread_exit_crit(mp); + vfs_op_thread_exit_crit(mp, mpcpu); return (cache_fpl_partial(fpl)); } if (!cache_fplookup_mp_supported(mp)) { - vfs_op_thread_exit_crit(mp); + vfs_op_thread_exit_crit(mp, mpcpu); return (cache_fpl_partial(fpl)); } vp = atomic_load_ptr(&mp->mnt_rootvnode); if (vp == NULL || VN_IS_DOOMED(vp)) { - vfs_op_thread_exit_crit(mp); + vfs_op_thread_exit_crit(mp, mpcpu); return (cache_fpl_partial(fpl)); } vp_seqc = vn_seqc_read_any(vp); if (seqc_in_modify(vp_seqc)) { - vfs_op_thread_exit_crit(mp); + vfs_op_thread_exit_crit(mp, mpcpu); return (cache_fpl_partial(fpl)); } prev_mp = mp; + prev_mpcpu = mpcpu; mp = atomic_load_ptr(&vp->v_mountedhere); if (mp == NULL) break; } - vfs_op_thread_exit_crit(prev_mp); + vfs_op_thread_exit_crit(prev_mp, prev_mpcpu); fpl->tvp = vp; fpl->tvp_seqc = vp_seqc; return (0); Modified: head/sys/kern/vfs_default.c ============================================================================== --- head/sys/kern/vfs_default.c Mon Nov 9 23:00:29 2020 (r367534) +++ head/sys/kern/vfs_default.c Mon Nov 9 23:02:13 2020 (r367535) @@ -663,6 +663,7 @@ vop_stdgetwritemount(ap) } */ *ap; { struct mount *mp; + struct mount_pcpu *mpcpu; struct vnode *vp; /* @@ -680,12 +681,12 @@ vop_stdgetwritemount(ap) *(ap->a_mpp) = NULL; return (0); } - if (vfs_op_thread_enter(mp)) { + if (vfs_op_thread_enter(mp, mpcpu)) { if (mp == vp->v_mount) { - vfs_mp_count_add_pcpu(mp, ref, 1); - vfs_op_thread_exit(mp); + vfs_mp_count_add_pcpu(mpcpu, ref, 1); + vfs_op_thread_exit(mp, mpcpu); } else { - vfs_op_thread_exit(mp); + vfs_op_thread_exit(mp, mpcpu); mp = NULL; } } else { Modified: head/sys/kern/vfs_mount.c ============================================================================== --- head/sys/kern/vfs_mount.c Mon Nov 9 23:00:29 2020 (r367534) +++ head/sys/kern/vfs_mount.c Mon Nov 9 23:02:13 2020 (r367535) @@ -127,14 +127,7 @@ mount_init(void *mem, int size, int flags) mtx_init(&mp->mnt_mtx, "struct mount mtx", NULL, MTX_DEF); mtx_init(&mp->mnt_listmtx, "struct mount vlist mtx", NULL, MTX_DEF); lockinit(&mp->mnt_explock, PVFS, "explock", 0, 0); - mp->mnt_thread_in_ops_pcpu = uma_zalloc_pcpu(pcpu_zone_4, - M_WAITOK | M_ZERO); - mp->mnt_ref_pcpu = uma_zalloc_pcpu(pcpu_zone_4, - M_WAITOK | M_ZERO); - mp->mnt_lockref_pcpu = uma_zalloc_pcpu(pcpu_zone_4, - M_WAITOK | M_ZERO); - mp->mnt_writeopcount_pcpu = uma_zalloc_pcpu(pcpu_zone_4, - M_WAITOK | M_ZERO); + mp->mnt_pcpu = uma_zalloc_pcpu(pcpu_zone_16, M_WAITOK | M_ZERO); mp->mnt_ref = 0; mp->mnt_vfs_ops = 1; mp->mnt_rootvnode = NULL; @@ -147,10 +140,7 @@ mount_fini(void *mem, int size) struct mount *mp; mp = (struct mount *)mem; - uma_zfree_pcpu(pcpu_zone_4, mp->mnt_writeopcount_pcpu); - uma_zfree_pcpu(pcpu_zone_4, mp->mnt_lockref_pcpu); - uma_zfree_pcpu(pcpu_zone_4, mp->mnt_ref_pcpu); - uma_zfree_pcpu(pcpu_zone_4, mp->mnt_thread_in_ops_pcpu); + uma_zfree_pcpu(pcpu_zone_16, mp->mnt_pcpu); lockdestroy(&mp->mnt_explock); mtx_destroy(&mp->mnt_listmtx); mtx_destroy(&mp->mnt_mtx); @@ -462,11 +452,12 @@ sys_nmount(struct thread *td, struct nmount_args *uap) void vfs_ref(struct mount *mp) { + struct mount_pcpu *mpcpu; CTR2(KTR_VFS, "%s: mp %p", __func__, mp); - if (vfs_op_thread_enter(mp)) { - vfs_mp_count_add_pcpu(mp, ref, 1); - vfs_op_thread_exit(mp); + if (vfs_op_thread_enter(mp, mpcpu)) { + vfs_mp_count_add_pcpu(mpcpu, ref, 1); + vfs_op_thread_exit(mp, mpcpu); return; } @@ -478,11 +469,12 @@ vfs_ref(struct mount *mp) void vfs_rel(struct mount *mp) { + struct mount_pcpu *mpcpu; CTR2(KTR_VFS, "%s: mp %p", __func__, mp); - if (vfs_op_thread_enter(mp)) { - vfs_mp_count_sub_pcpu(mp, ref, 1); - vfs_op_thread_exit(mp); + if (vfs_op_thread_enter(mp, mpcpu)) { + vfs_mp_count_sub_pcpu(mpcpu, ref, 1); + vfs_op_thread_exit(mp, mpcpu); return; } @@ -503,6 +495,12 @@ vfs_mount_alloc(struct vnode *vp, struct vfsconf *vfsp mp = uma_zalloc(mount_zone, M_WAITOK); bzero(&mp->mnt_startzero, __rangeof(struct mount, mnt_startzero, mnt_endzero)); + mp->mnt_kern_flag = 0; + mp->mnt_flag = 0; + mp->mnt_rootvnode = NULL; + mp->mnt_vnodecovered = NULL; + mp->mnt_op = NULL; + mp->mnt_vfc = NULL; TAILQ_INIT(&mp->mnt_nvnodelist); mp->mnt_nvnodelistsize = 0; TAILQ_INIT(&mp->mnt_lazyvnodelist); @@ -1513,6 +1511,7 @@ dounmount_cleanup(struct mount *mp, struct vnode *cove void vfs_op_enter(struct mount *mp) { + struct mount_pcpu *mpcpu; int cpu; MNT_ILOCK(mp); @@ -1523,12 +1522,16 @@ vfs_op_enter(struct mount *mp) } vfs_op_barrier_wait(mp); CPU_FOREACH(cpu) { - mp->mnt_ref += - zpcpu_replace_cpu(mp->mnt_ref_pcpu, 0, cpu); - mp->mnt_lockref += - zpcpu_replace_cpu(mp->mnt_lockref_pcpu, 0, cpu); - mp->mnt_writeopcount += - zpcpu_replace_cpu(mp->mnt_writeopcount_pcpu, 0, cpu); + mpcpu = vfs_mount_pcpu_remote(mp, cpu); + + mp->mnt_ref += mpcpu->mntp_ref; + mpcpu->mntp_ref = 0; + + mp->mnt_lockref += mpcpu->mntp_lockref; + mpcpu->mntp_lockref = 0; + + mp->mnt_writeopcount += mpcpu->mntp_writeopcount; + mpcpu->mntp_writeopcount = 0; } if (mp->mnt_ref <= 0 || mp->mnt_lockref < 0 || mp->mnt_writeopcount < 0) panic("%s: invalid count(s) on mp %p: ref %d lockref %d writeopcount %d\n", @@ -1581,13 +1584,13 @@ vfs_op_wait_func(void *arg, int cpu) { struct vfs_op_barrier_ipi *vfsopipi; struct mount *mp; - int *in_op; + struct mount_pcpu *mpcpu; vfsopipi = __containerof(arg, struct vfs_op_barrier_ipi, srcra); mp = vfsopipi->mp; - in_op = zpcpu_get_cpu(mp->mnt_thread_in_ops_pcpu, cpu); - while (atomic_load_int(in_op)) + mpcpu = vfs_mount_pcpu_remote(mp, cpu); + while (atomic_load_int(&mpcpu->mntp_thread_in_ops)) cpu_spinwait(); } @@ -1610,15 +1613,17 @@ vfs_op_barrier_wait(struct mount *mp) void vfs_assert_mount_counters(struct mount *mp) { + struct mount_pcpu *mpcpu; int cpu; if (mp->mnt_vfs_ops == 0) return; CPU_FOREACH(cpu) { - if (*zpcpu_get_cpu(mp->mnt_ref_pcpu, cpu) != 0 || - *zpcpu_get_cpu(mp->mnt_lockref_pcpu, cpu) != 0 || - *zpcpu_get_cpu(mp->mnt_writeopcount_pcpu, cpu) != 0) + mpcpu = vfs_mount_pcpu_remote(mp, cpu); + if (mpcpu->mntp_ref != 0 || + mpcpu->mntp_lockref != 0 || + mpcpu->mntp_writeopcount != 0) vfs_dump_mount_counters(mp); } } @@ -1626,33 +1631,34 @@ vfs_assert_mount_counters(struct mount *mp) void vfs_dump_mount_counters(struct mount *mp) { - int cpu, *count; + struct mount_pcpu *mpcpu; int ref, lockref, writeopcount; + int cpu; printf("%s: mp %p vfs_ops %d\n", __func__, mp, mp->mnt_vfs_ops); printf(" ref : "); ref = mp->mnt_ref; CPU_FOREACH(cpu) { - count = zpcpu_get_cpu(mp->mnt_ref_pcpu, cpu); - printf("%d ", *count); - ref += *count; + mpcpu = vfs_mount_pcpu_remote(mp, cpu); + printf("%d ", mpcpu->mntp_ref); + ref += mpcpu->mntp_ref; } printf("\n"); printf(" lockref : "); lockref = mp->mnt_lockref; CPU_FOREACH(cpu) { - count = zpcpu_get_cpu(mp->mnt_lockref_pcpu, cpu); - printf("%d ", *count); - lockref += *count; + mpcpu = vfs_mount_pcpu_remote(mp, cpu); + printf("%d ", mpcpu->mntp_lockref); + lockref += mpcpu->mntp_lockref; } printf("\n"); printf("writeopcount: "); writeopcount = mp->mnt_writeopcount; CPU_FOREACH(cpu) { - count = zpcpu_get_cpu(mp->mnt_writeopcount_pcpu, cpu); - printf("%d ", *count); - writeopcount += *count; + mpcpu = vfs_mount_pcpu_remote(mp, cpu); + printf("%d ", mpcpu->mntp_writeopcount); + writeopcount += mpcpu->mntp_writeopcount; } printf("\n"); @@ -1668,27 +1674,34 @@ vfs_dump_mount_counters(struct mount *mp) int vfs_mount_fetch_counter(struct mount *mp, enum mount_counter which) { - int *base, *pcpu; + struct mount_pcpu *mpcpu; int cpu, sum; switch (which) { case MNT_COUNT_REF: - base = &mp->mnt_ref; - pcpu = mp->mnt_ref_pcpu; + sum = mp->mnt_ref; break; case MNT_COUNT_LOCKREF: - base = &mp->mnt_lockref; - pcpu = mp->mnt_lockref_pcpu; + sum = mp->mnt_lockref; break; case MNT_COUNT_WRITEOPCOUNT: - base = &mp->mnt_writeopcount; - pcpu = mp->mnt_writeopcount_pcpu; + sum = mp->mnt_writeopcount; break; } - sum = *base; CPU_FOREACH(cpu) { - sum += *zpcpu_get_cpu(pcpu, cpu); + mpcpu = vfs_mount_pcpu_remote(mp, cpu); + switch (which) { + case MNT_COUNT_REF: + sum += mpcpu->mntp_ref; + break; + case MNT_COUNT_LOCKREF: + sum += mpcpu->mntp_lockref; + break; + case MNT_COUNT_WRITEOPCOUNT: + sum += mpcpu->mntp_writeopcount; + break; + } } return (sum); } Modified: head/sys/kern/vfs_subr.c ============================================================================== --- head/sys/kern/vfs_subr.c Mon Nov 9 23:00:29 2020 (r367534) +++ head/sys/kern/vfs_subr.c Mon Nov 9 23:02:13 2020 (r367535) @@ -734,17 +734,18 @@ SYSINIT(vfs, SI_SUB_VFS, SI_ORDER_FIRST, vntblinit, NU int vfs_busy(struct mount *mp, int flags) { + struct mount_pcpu *mpcpu; MPASS((flags & ~MBF_MASK) == 0); CTR3(KTR_VFS, "%s: mp %p with flags %d", __func__, mp, flags); - if (vfs_op_thread_enter(mp)) { + if (vfs_op_thread_enter(mp, mpcpu)) { MPASS((mp->mnt_kern_flag & MNTK_DRAINING) == 0); MPASS((mp->mnt_kern_flag & MNTK_UNMOUNT) == 0); MPASS((mp->mnt_kern_flag & MNTK_REFEXPIRE) == 0); - vfs_mp_count_add_pcpu(mp, ref, 1); - vfs_mp_count_add_pcpu(mp, lockref, 1); - vfs_op_thread_exit(mp); + vfs_mp_count_add_pcpu(mpcpu, ref, 1); + vfs_mp_count_add_pcpu(mpcpu, lockref, 1); + vfs_op_thread_exit(mp, mpcpu); if (flags & MBF_MNTLSTLOCK) mtx_unlock(&mountlist_mtx); return (0); @@ -794,15 +795,16 @@ vfs_busy(struct mount *mp, int flags) void vfs_unbusy(struct mount *mp) { + struct mount_pcpu *mpcpu; int c; CTR2(KTR_VFS, "%s: mp %p", __func__, mp); - if (vfs_op_thread_enter(mp)) { + if (vfs_op_thread_enter(mp, mpcpu)) { MPASS((mp->mnt_kern_flag & MNTK_DRAINING) == 0); - vfs_mp_count_sub_pcpu(mp, lockref, 1); - vfs_mp_count_sub_pcpu(mp, ref, 1); - vfs_op_thread_exit(mp); + vfs_mp_count_sub_pcpu(mpcpu, lockref, 1); + vfs_mp_count_sub_pcpu(mpcpu, ref, 1); + vfs_op_thread_exit(mp, mpcpu); return; } @@ -6399,18 +6401,19 @@ restart: int vfs_cache_root(struct mount *mp, int flags, struct vnode **vpp) { + struct mount_pcpu *mpcpu; struct vnode *vp; int error; - if (!vfs_op_thread_enter(mp)) + if (!vfs_op_thread_enter(mp, mpcpu)) return (vfs_cache_root_fallback(mp, flags, vpp)); vp = atomic_load_ptr(&mp->mnt_rootvnode); if (vp == NULL || VN_IS_DOOMED(vp)) { - vfs_op_thread_exit(mp); + vfs_op_thread_exit(mp, mpcpu); return (vfs_cache_root_fallback(mp, flags, vpp)); } vrefact(vp); - vfs_op_thread_exit(mp); + vfs_op_thread_exit(mp, mpcpu); error = vn_lock(vp, flags); if (error != 0) { vrele(vp); Modified: head/sys/kern/vfs_vnops.c ============================================================================== --- head/sys/kern/vfs_vnops.c Mon Nov 9 23:00:29 2020 (r367534) +++ head/sys/kern/vfs_vnops.c Mon Nov 9 23:02:13 2020 (r367535) @@ -1761,13 +1761,14 @@ vn_closefile(struct file *fp, struct thread *td) static int vn_start_write_refed(struct mount *mp, int flags, bool mplocked) { + struct mount_pcpu *mpcpu; int error, mflags; if (__predict_true(!mplocked) && (flags & V_XSLEEP) == 0 && - vfs_op_thread_enter(mp)) { + vfs_op_thread_enter(mp, mpcpu)) { MPASS((mp->mnt_kern_flag & MNTK_SUSPEND) == 0); - vfs_mp_count_add_pcpu(mp, writeopcount, 1); - vfs_op_thread_exit(mp); + vfs_mp_count_add_pcpu(mpcpu, writeopcount, 1); + vfs_op_thread_exit(mp, mpcpu); return (0); } @@ -1917,15 +1918,16 @@ vn_start_secondary_write(struct vnode *vp, struct moun void vn_finished_write(struct mount *mp) { + struct mount_pcpu *mpcpu; int c; if (mp == NULL) return; - if (vfs_op_thread_enter(mp)) { - vfs_mp_count_sub_pcpu(mp, writeopcount, 1); - vfs_mp_count_sub_pcpu(mp, ref, 1); - vfs_op_thread_exit(mp); + if (vfs_op_thread_enter(mp, mpcpu)) { + vfs_mp_count_sub_pcpu(mpcpu, writeopcount, 1); + vfs_mp_count_sub_pcpu(mpcpu, ref, 1); + vfs_op_thread_exit(mp, mpcpu); return; } Modified: head/sys/sys/mount.h ============================================================================== --- head/sys/sys/mount.h Mon Nov 9 23:00:29 2020 (r367534) +++ head/sys/sys/mount.h Mon Nov 9 23:02:13 2020 (r367535) @@ -177,6 +177,16 @@ struct vfsopt { int seen; }; +struct mount_pcpu { + int mntp_thread_in_ops; + int mntp_ref; + int mntp_lockref; + int mntp_writeopcount; +}; + +_Static_assert(sizeof(struct mount_pcpu) == 16, + "the struct is allocated from pcpu 16 zone"); + /* * Structure per mounted filesystem. Each mounted filesystem has an * array of operations and an instance record. The filesystems are @@ -192,20 +202,23 @@ struct vfsopt { * */ struct mount { - struct mtx mnt_mtx; /* mount structure interlock */ + int mnt_vfs_ops; /* (i) pending vfs ops */ + int mnt_kern_flag; /* (i) kernel only flags */ + uint64_t mnt_flag; /* (i) flags shared with user */ + struct mount_pcpu *mnt_pcpu; /* per-CPU data */ + struct vnode *mnt_rootvnode; + struct vnode *mnt_vnodecovered; /* vnode we mounted on */ + struct vfsops *mnt_op; /* operations on fs */ + struct vfsconf *mnt_vfc; /* configuration info */ + struct mtx __aligned(CACHE_LINE_SIZE) mnt_mtx; /* mount structure interlock */ int mnt_gen; /* struct mount generation */ #define mnt_startzero mnt_list TAILQ_ENTRY(mount) mnt_list; /* (m) mount list */ - struct vfsops *mnt_op; /* operations on fs */ - struct vfsconf *mnt_vfc; /* configuration info */ - struct vnode *mnt_vnodecovered; /* vnode we mounted on */ struct vnode *mnt_syncer; /* syncer vnode */ int mnt_ref; /* (i) Reference count */ struct vnodelst mnt_nvnodelist; /* (i) list of vnodes */ int mnt_nvnodelistsize; /* (i) # of vnodes */ int mnt_writeopcount; /* (i) write syscalls pending */ - int mnt_kern_flag; /* (i) kernel only flags */ - uint64_t mnt_flag; /* (i) flags shared with user */ struct vfsoptlist *mnt_opt; /* current mount options */ struct vfsoptlist *mnt_optnew; /* new options passed to fs */ int mnt_maxsymlinklen; /* max size of short symlink */ @@ -229,12 +242,6 @@ struct mount { struct lock mnt_explock; /* vfs_export walkers lock */ TAILQ_ENTRY(mount) mnt_upper_link; /* (m) we in the all uppers */ TAILQ_HEAD(, mount) mnt_uppers; /* (m) upper mounts over us*/ - int __aligned(CACHE_LINE_SIZE) mnt_vfs_ops;/* (i) pending vfs ops */ - int *mnt_thread_in_ops_pcpu; - int *mnt_ref_pcpu; - int *mnt_lockref_pcpu; - int *mnt_writeopcount_pcpu; - struct vnode *mnt_rootvnode; }; /* @@ -1054,7 +1061,7 @@ void resume_all_fs(void); /* * Code transitioning mnt_vfs_ops to > 0 issues IPIs until it observes - * all CPUs not executing code enclosed by mnt_thread_in_ops_pcpu. + * all CPUs not executing code enclosed by thread_in_ops_pcpu variable. * * This provides an invariant that by the time the last CPU is observed not * executing, everyone else entering will see the counter > 0 and exit. @@ -1064,52 +1071,58 @@ void resume_all_fs(void); * before making any changes or only make changes safe while the section is * executed. */ +#define vfs_mount_pcpu(mp) zpcpu_get(mp->mnt_pcpu) +#define vfs_mount_pcpu_remote(mp, cpu) zpcpu_get_cpu(mp->mnt_pcpu, cpu) + #define vfs_op_thread_entered(mp) ({ \ MPASS(curthread->td_critnest > 0); \ - *zpcpu_get(mp->mnt_thread_in_ops_pcpu) == 1; \ + struct mount_pcpu *_mpcpu = vfs_mount_pcpu(mp); \ + _mpcpu->mntp_thread_in_ops == 1; \ }) -#define vfs_op_thread_enter_crit(mp) ({ \ +#define vfs_op_thread_enter_crit(mp, _mpcpu) ({ \ bool _retval_crit = true; \ MPASS(curthread->td_critnest > 0); \ - MPASS(!vfs_op_thread_entered(mp)); \ - zpcpu_set_protected(mp->mnt_thread_in_ops_pcpu, 1); \ + _mpcpu = vfs_mount_pcpu(mp); \ + MPASS(mpcpu->mntp_thread_in_ops == 0); \ + _mpcpu->mntp_thread_in_ops = 1; \ __compiler_membar(); \ if (__predict_false(mp->mnt_vfs_ops > 0)) { \ - vfs_op_thread_exit_crit(mp); \ + vfs_op_thread_exit_crit(mp, _mpcpu); \ _retval_crit = false; \ } \ _retval_crit; \ }) -#define vfs_op_thread_enter(mp) ({ \ +#define vfs_op_thread_enter(mp, _mpcpu) ({ \ bool _retval; \ critical_enter(); \ - _retval = vfs_op_thread_enter_crit(mp); \ + _retval = vfs_op_thread_enter_crit(mp, _mpcpu); \ if (__predict_false(!_retval)) \ critical_exit(); \ _retval; \ }) -#define vfs_op_thread_exit_crit(mp) do { \ - MPASS(vfs_op_thread_entered(mp)); \ +#define vfs_op_thread_exit_crit(mp, _mpcpu) do { \ + MPASS(_mpcpu == vfs_mount_pcpu(mp)); \ + MPASS(_mpcpu->mntp_thread_in_ops == 1); \ __compiler_membar(); \ - zpcpu_set_protected(mp->mnt_thread_in_ops_pcpu, 0); \ + _mpcpu->mntp_thread_in_ops = 0; \ } while (0) -#define vfs_op_thread_exit(mp) do { \ - vfs_op_thread_exit_crit(mp); \ +#define vfs_op_thread_exit(mp, _mpcpu) do { \ + vfs_op_thread_exit_crit(mp, _mpcpu); \ critical_exit(); \ } while (0) -#define vfs_mp_count_add_pcpu(mp, count, val) do { \ - MPASS(vfs_op_thread_entered(mp)); \ - zpcpu_add_protected(mp->mnt_##count##_pcpu, val); \ +#define vfs_mp_count_add_pcpu(_mpcpu, count, val) do { \ + MPASS(_mpcpu->mntp_thread_in_ops == 1); \ + _mpcpu->mntp_##count += val; \ } while (0) -#define vfs_mp_count_sub_pcpu(mp, count, val) do { \ - MPASS(vfs_op_thread_entered(mp)); \ - zpcpu_sub_protected(mp->mnt_##count##_pcpu, val); \ +#define vfs_mp_count_sub_pcpu(_mpcpu, count, val) do { \ + MPASS(_mpcpu->mntp_thread_in_ops == 1); \ + _mpcpu->mntp_##count -= val; \ } while (0) #else /* !_KERNEL */
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?202011092302.0A9N2D63015034>