From owner-svn-src-user@FreeBSD.ORG Thu Oct 10 19:03:36 2013 Return-Path: Delivered-To: svn-src-user@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [8.8.178.115]) (using TLSv1 with cipher ADH-AES256-SHA (256/256 bits)) (No client certificate requested) by hub.freebsd.org (Postfix) with ESMTP id AE5B57A1; Thu, 10 Oct 2013 19:03:36 +0000 (UTC) (envelope-from andre@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:1900:2254:2068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by mx1.freebsd.org (Postfix) with ESMTPS id 9AC2424CD; Thu, 10 Oct 2013 19:03:36 +0000 (UTC) Received: from svn.freebsd.org ([127.0.1.70]) by svn.freebsd.org (8.14.7/8.14.7) with ESMTP id r9AJ3a9G008056; Thu, 10 Oct 2013 19:03:36 GMT (envelope-from andre@svn.freebsd.org) Received: (from andre@localhost) by svn.freebsd.org (8.14.7/8.14.5/Submit) id r9AJ3aWX008053; Thu, 10 Oct 2013 19:03:36 GMT (envelope-from andre@svn.freebsd.org) Message-Id: <201310101903.r9AJ3aWX008053@svn.freebsd.org> From: Andre Oppermann Date: Thu, 10 Oct 2013 19:03:36 +0000 (UTC) To: src-committers@freebsd.org, svn-src-user@freebsd.org Subject: svn commit: r256287 - in user/andre/mbuf_staging: kern sys X-SVN-Group: user MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-user@freebsd.org X-Mailman-Version: 2.1.14 Precedence: list List-Id: "SVN commit messages for the experimental " user" src tree" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 10 Oct 2013 19:03:36 -0000 Author: andre Date: Thu Oct 10 19:03:35 2013 New Revision: 256287 URL: http://svnweb.freebsd.org/changeset/base/256287 Log: De-inline the mbuf and cluster allocation functions and mechanically move them to kern/kern_mbuf.c with exception of m_extaddref() which going to kern/uipc_mbuf.c. Having all mbuf and cluster allocation functions as real functions again gives us significant future freedom in optimizing the backend allocator without breaking the ABI for drivers. As a side-note the function call overhead on modern CPUs is very low and in other cases micro-benchmarks have shown that de-inlining actually slightly improved performance by reducing the number of instructions and improving I-cache efficiency. This change hasn't been measured yet though. Discussed with: glebius Modified: user/andre/mbuf_staging/kern/kern_mbuf.c user/andre/mbuf_staging/kern/uipc_mbuf.c user/andre/mbuf_staging/sys/mbuf.h Modified: user/andre/mbuf_staging/kern/kern_mbuf.c ============================================================================== --- user/andre/mbuf_staging/kern/kern_mbuf.c Thu Oct 10 18:34:46 2013 (r256286) +++ user/andre/mbuf_staging/kern/kern_mbuf.c Thu Oct 10 19:03:35 2013 (r256287) @@ -640,6 +640,60 @@ mb_ctor_pack(void *mem, int size, void * return (error); } +uma_zone_t +m_getzone(int size) +{ + uma_zone_t zone; + + switch (size) { + case MCLBYTES: + zone = zone_clust; + break; +#if MJUMPAGESIZE != MCLBYTES + case MJUMPAGESIZE: + zone = zone_jumbop; + break; +#endif + case MJUM9BYTES: + zone = zone_jumbo9; + break; + case MJUM16BYTES: + zone = zone_jumbo16; + break; + default: + panic("%s: invalid cluster size", __func__); + } + + return (zone); +} + +/* + * Initialize an mbuf with linear storage. + * + * Inline because the consumer text overhead will be roughly the same to + * initialize or call a function with this many parameters and M_PKTHDR + * should go away with constant propagation for !MGETHDR. + */ +int +m_init(struct mbuf *m, uma_zone_t zone, int size, int how, short type, + int flags) +{ + int error; + + m->m_next = NULL; + m->m_nextpkt = NULL; + m->m_data = m->m_dat; + m->m_len = 0; + m->m_flags = flags; + m->m_type = type; + if (flags & M_PKTHDR) { + if ((error = m_pkthdr_init(m, how)) != 0) + return (error); + } + + return (0); +} + int m_pkthdr_init(struct mbuf *m, int how) { @@ -671,6 +725,132 @@ m_pkthdr_init(struct mbuf *m, int how) return (0); } +struct mbuf * +m_get(int how, short type) +{ + struct mb_args args; + + args.flags = 0; + args.type = type; + return (uma_zalloc_arg(zone_mbuf, &args, how)); +} + +/* + * XXX This should be deprecated, very little use. + */ +struct mbuf * +m_getclr(int how, short type) +{ + struct mbuf *m; + struct mb_args args; + + args.flags = 0; + args.type = type; + m = uma_zalloc_arg(zone_mbuf, &args, how); + if (m != NULL) + bzero(m->m_data, MLEN); + return (m); +} + +struct mbuf * +m_gethdr(int how, short type) +{ + struct mb_args args; + + args.flags = M_PKTHDR; + args.type = type; + return (uma_zalloc_arg(zone_mbuf, &args, how)); +} + +struct mbuf * +m_getcl(int how, short type, int flags) +{ + struct mb_args args; + + args.flags = flags; + args.type = type; + return (uma_zalloc_arg(zone_pack, &args, how)); +} + +void +m_clget(struct mbuf *m, int how) +{ + + if (m->m_flags & M_EXT) + printf("%s: %p mbuf already has cluster\n", __func__, m); + m->m_ext.ext_buf = (char *)NULL; + uma_zalloc_arg(zone_clust, m, how); + /* + * On a cluster allocation failure, drain the packet zone and retry, + * we might be able to loosen a few clusters up on the drain. + */ + if ((how & M_NOWAIT) && (m->m_ext.ext_buf == NULL)) { + zone_drain(zone_pack); + uma_zalloc_arg(zone_clust, m, how); + } +} + +/* + * m_cljget() is different from m_clget() as it can allocate clusters without + * attaching them to an mbuf. In that case the return value is the pointer + * to the cluster of the requested size. If an mbuf was specified, it gets + * the cluster attached to it and the return value can be safely ignored. + * For size it takes MCLBYTES, MJUMPAGESIZE, MJUM9BYTES, MJUM16BYTES. + */ +void * +m_cljget(struct mbuf *m, int how, int size) +{ + uma_zone_t zone; + + if (m && m->m_flags & M_EXT) + printf("%s: %p mbuf already has cluster\n", __func__, m); + if (m != NULL) + m->m_ext.ext_buf = NULL; + + zone = m_getzone(size); + return (uma_zalloc_arg(zone, m, how)); +} + +void +m_cljset(struct mbuf *m, void *cl, int type) +{ + uma_zone_t zone; + int size; + + switch (type) { + case EXT_CLUSTER: + size = MCLBYTES; + zone = zone_clust; + break; +#if MJUMPAGESIZE != MCLBYTES + case EXT_JUMBOP: + size = MJUMPAGESIZE; + zone = zone_jumbop; + break; +#endif + case EXT_JUMBO9: + size = MJUM9BYTES; + zone = zone_jumbo9; + break; + case EXT_JUMBO16: + size = MJUM16BYTES; + zone = zone_jumbo16; + break; + default: + panic("%s: unknown cluster type", __func__); + break; + } + + m->m_data = m->m_ext.ext_buf = cl; + m->m_ext.ext_free = m->m_ext.ext_arg1 = m->m_ext.ext_arg2 = NULL; + m->m_ext.ext_size = size; + m->m_ext.ext_type = type; + m->m_ext.ext_flags = 0; + m->m_ext.ref_cnt = uma_find_refcnt(zone, cl); + m->m_flags |= M_EXT; + +} + /* * This is the protocol drain routine. * Modified: user/andre/mbuf_staging/kern/uipc_mbuf.c ============================================================================== --- user/andre/mbuf_staging/kern/uipc_mbuf.c Thu Oct 10 18:34:46 2013 (r256286) +++ user/andre/mbuf_staging/kern/uipc_mbuf.c Thu Oct 10 19:03:35 2013 (r256287) @@ -281,6 +281,28 @@ m_extadd(struct mbuf *mb, caddr_t buf, u } /* + * Associated an external reference counted buffer with an mbuf. + */ +void +m_extaddref(struct mbuf *m, caddr_t buf, u_int size, u_int *ref_cnt, + int (*freef)(struct mbuf *, void *, void *), void *arg1, void *arg2) +{ + + KASSERT(ref_cnt != NULL, ("%s: ref_cnt not provided", __func__)); + + atomic_add_int(ref_cnt, 1); + m->m_flags |= M_EXT; + m->m_ext.ext_buf = buf; + m->m_ext.ref_cnt = ref_cnt; + m->m_data = m->m_ext.ext_buf; + m->m_ext.ext_size = size; + m->m_ext.ext_free = freef; + m->m_ext.ext_arg1 = arg1; + m->m_ext.ext_arg2 = arg2; + m->m_ext.ext_type = EXT_EXTREF; +} + +/* * Non-directly-exported function to clean up after mbufs with M_EXT * storage attached to them if the reference count hits 1. */ Modified: user/andre/mbuf_staging/sys/mbuf.h ============================================================================== --- user/andre/mbuf_staging/sys/mbuf.h Thu Oct 10 18:34:46 2013 (r256286) +++ user/andre/mbuf_staging/sys/mbuf.h Thu Oct 10 19:03:35 2013 (r256287) @@ -516,238 +516,19 @@ extern uma_zone_t zone_ext_refcnt; void mb_free_ext(struct mbuf *); int m_pkthdr_init(struct mbuf *, int); - -static __inline int -m_gettype(int size) -{ - int type; - - switch (size) { - case MSIZE: - type = EXT_MBUF; - break; - case MCLBYTES: - type = EXT_CLUSTER; - break; -#if MJUMPAGESIZE != MCLBYTES - case MJUMPAGESIZE: - type = EXT_JUMBOP; - break; -#endif - case MJUM9BYTES: - type = EXT_JUMBO9; - break; - case MJUM16BYTES: - type = EXT_JUMBO16; - break; - default: - panic("%s: invalid cluster size", __func__); - } - - return (type); -} - -/* - * Associated an external reference counted buffer with an mbuf. - */ -static __inline void -m_extaddref(struct mbuf *m, caddr_t buf, u_int size, u_int *ref_cnt, - int (*freef)(struct mbuf *, void *, void *), void *arg1, void *arg2) -{ - - KASSERT(ref_cnt != NULL, ("%s: ref_cnt not provided", __func__)); - - atomic_add_int(ref_cnt, 1); - m->m_flags |= M_EXT; - m->m_ext.ext_buf = buf; - m->m_ext.ref_cnt = ref_cnt; - m->m_data = m->m_ext.ext_buf; - m->m_ext.ext_size = size; - m->m_ext.ext_free = freef; - m->m_ext.ext_arg1 = arg1; - m->m_ext.ext_arg2 = arg2; - m->m_ext.ext_type = EXT_EXTREF; -} - -static __inline uma_zone_t -m_getzone(int size) -{ - uma_zone_t zone; - - switch (size) { - case MCLBYTES: - zone = zone_clust; - break; -#if MJUMPAGESIZE != MCLBYTES - case MJUMPAGESIZE: - zone = zone_jumbop; - break; -#endif - case MJUM9BYTES: - zone = zone_jumbo9; - break; - case MJUM16BYTES: - zone = zone_jumbo16; - break; - default: - panic("%s: invalid cluster size", __func__); - } - - return (zone); -} - -/* - * Initialize an mbuf with linear storage. - * - * Inline because the consumer text overhead will be roughly the same to - * initialize or call a function with this many parameters and M_PKTHDR - * should go away with constant propagation for !MGETHDR. - */ -static __inline int -m_init(struct mbuf *m, uma_zone_t zone, int size, int how, short type, - int flags) -{ - int error; - - m->m_next = NULL; - m->m_nextpkt = NULL; - m->m_data = m->m_dat; - m->m_len = 0; - m->m_flags = flags; - m->m_type = type; - if (flags & M_PKTHDR) { - if ((error = m_pkthdr_init(m, how)) != 0) - return (error); - } - - return (0); -} - -static __inline struct mbuf * -m_get(int how, short type) -{ - struct mb_args args; - - args.flags = 0; - args.type = type; - return (uma_zalloc_arg(zone_mbuf, &args, how)); -} - -/* - * XXX This should be deprecated, very little use. - */ -static __inline struct mbuf * -m_getclr(int how, short type) -{ - struct mbuf *m; - struct mb_args args; - - args.flags = 0; - args.type = type; - m = uma_zalloc_arg(zone_mbuf, &args, how); - if (m != NULL) - bzero(m->m_data, MLEN); - return (m); -} - -static __inline struct mbuf * -m_gethdr(int how, short type) -{ - struct mb_args args; - - args.flags = M_PKTHDR; - args.type = type; - return (uma_zalloc_arg(zone_mbuf, &args, how)); -} - -static __inline struct mbuf * -m_getcl(int how, short type, int flags) -{ - struct mb_args args; - - args.flags = flags; - args.type = type; - return (uma_zalloc_arg(zone_pack, &args, how)); -} - -static __inline void -m_clget(struct mbuf *m, int how) -{ - - if (m->m_flags & M_EXT) - printf("%s: %p mbuf already has cluster\n", __func__, m); - m->m_ext.ext_buf = (char *)NULL; - uma_zalloc_arg(zone_clust, m, how); - /* - * On a cluster allocation failure, drain the packet zone and retry, - * we might be able to loosen a few clusters up on the drain. - */ - if ((how & M_NOWAIT) && (m->m_ext.ext_buf == NULL)) { - zone_drain(zone_pack); - uma_zalloc_arg(zone_clust, m, how); - } -} - -/* - * m_cljget() is different from m_clget() as it can allocate clusters without - * attaching them to an mbuf. In that case the return value is the pointer - * to the cluster of the requested size. If an mbuf was specified, it gets - * the cluster attached to it and the return value can be safely ignored. - * For size it takes MCLBYTES, MJUMPAGESIZE, MJUM9BYTES, MJUM16BYTES. - */ -static __inline void * -m_cljget(struct mbuf *m, int how, int size) -{ - uma_zone_t zone; - - if (m && m->m_flags & M_EXT) - printf("%s: %p mbuf already has cluster\n", __func__, m); - if (m != NULL) - m->m_ext.ext_buf = NULL; - - zone = m_getzone(size); - return (uma_zalloc_arg(zone, m, how)); -} - -static __inline void -m_cljset(struct mbuf *m, void *cl, int type) -{ - uma_zone_t zone; - int size; - - switch (type) { - case EXT_CLUSTER: - size = MCLBYTES; - zone = zone_clust; - break; -#if MJUMPAGESIZE != MCLBYTES - case EXT_JUMBOP: - size = MJUMPAGESIZE; - zone = zone_jumbop; - break; -#endif - case EXT_JUMBO9: - size = MJUM9BYTES; - zone = zone_jumbo9; - break; - case EXT_JUMBO16: - size = MJUM16BYTES; - zone = zone_jumbo16; - break; - default: - panic("%s: unknown cluster type", __func__); - break; - } - - m->m_data = m->m_ext.ext_buf = cl; - m->m_ext.ext_free = m->m_ext.ext_arg1 = m->m_ext.ext_arg2 = NULL; - m->m_ext.ext_size = size; - m->m_ext.ext_type = type; - m->m_ext.ext_flags = 0; - m->m_ext.ref_cnt = uma_find_refcnt(zone, cl); - m->m_flags |= M_EXT; - -} +uma_zone_t m_getzone(int size); +int m_gettype(int); +void m_extaddref(struct mbuf *, caddr_t, u_int, u_int *, + int (*)(struct mbuf *, void *, void *), void *, void *); +uma_zone_t m_getzone(int); +int m_init(struct mbuf *, uma_zone_t, int, int, short, int); +struct mbuf *m_get(int, short); +struct mbuf *m_getclr(int, short); +struct mbuf *m_gethdr(int, short); +struct mbuf *m_getcl(int, short, int); +void m_clget(struct mbuf *, int); +void *m_cljget(struct mbuf *, int, int); +void m_cljset(struct mbuf *, void *, int); static __inline void m_chtype(struct mbuf *m, short new_type)