From owner-p4-projects@FreeBSD.ORG Sun Apr 1 20:12:43 2007 Return-Path: X-Original-To: p4-projects@freebsd.org Delivered-To: p4-projects@freebsd.org Received: by hub.freebsd.org (Postfix, from userid 32767) id 888B816A40F; Sun, 1 Apr 2007 20:12:43 +0000 (UTC) X-Original-To: perforce@freebsd.org Delivered-To: perforce@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [69.147.83.52]) by hub.freebsd.org (Postfix) with ESMTP id 12ECB16A403 for ; Sun, 1 Apr 2007 20:12:43 +0000 (UTC) (envelope-from kmacy@freebsd.org) Received: from repoman.freebsd.org (repoman.freebsd.org [69.147.83.41]) by mx1.freebsd.org (Postfix) with ESMTP id EC34D13C4C9 for ; Sun, 1 Apr 2007 20:12:42 +0000 (UTC) (envelope-from kmacy@freebsd.org) Received: from repoman.freebsd.org (localhost [127.0.0.1]) by repoman.freebsd.org (8.13.8/8.13.8) with ESMTP id l31KCgFU096877 for ; Sun, 1 Apr 2007 20:12:42 GMT (envelope-from kmacy@freebsd.org) Received: (from perforce@localhost) by repoman.freebsd.org (8.13.8/8.13.8/Submit) id l31KCgMg096871 for perforce@freebsd.org; Sun, 1 Apr 2007 20:12:42 GMT (envelope-from kmacy@freebsd.org) Date: Sun, 1 Apr 2007 20:12:42 GMT Message-Id: <200704012012.l31KCgMg096871@repoman.freebsd.org> X-Authentication-Warning: repoman.freebsd.org: perforce set sender to kmacy@freebsd.org using -f From: Kip Macy To: Perforce Change Reviews Cc: Subject: PERFORCE change 117121 for review X-BeenThere: p4-projects@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: p4 projects tree changes List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Sun, 01 Apr 2007 20:12:43 -0000 http://perforce.freebsd.org/chv.cgi?CH=117121 Change 117121 by kmacy@kmacy_vt-x:opentoe_init on 2007/04/01 20:11:47 fix mb_ctor_clust to actually reference the correct zone for a cluster eliminate duplicate calls to uma_find_refcnt cleanup the INVARIANT checks in mb_dtor_clust add m_cljset to attach a cluster to a deferred mbuf add m_getzone to eliminate duplicated zone lookup code for clusters add mbuf_iovec for using pktdat for cache-friendly scatter-gather of clusters Affected files ... .. //depot/projects/opentoe/sys/kern/kern_mbuf.c#3 edit .. //depot/projects/opentoe/sys/sys/mbuf.h#3 edit Differences ... ==== //depot/projects/opentoe/sys/kern/kern_mbuf.c#3 (text+ko) ==== @@ -423,7 +423,11 @@ default: panic("unknown cluster size"); break; - } + } + + refcnt = uma_find_refcnt(zone, mem); + *refcnt = 1; + m = (struct mbuf *)arg; if (m != NULL) { m->m_ext.ext_buf = (caddr_t)mem; @@ -433,12 +437,9 @@ m->m_ext.ext_args = NULL; m->m_ext.ext_size = size; m->m_ext.ext_type = type; - m->m_ext.ref_cnt = uma_find_refcnt(zone, mem); - *m->m_ext.ref_cnt = 1; - } else { - refcnt = uma_find_refcnt(zone, mem); - *refcnt = 1; + m->m_ext.ref_cnt = refcnt; } + return (0); } @@ -448,11 +449,13 @@ static void mb_dtor_clust(void *mem, int size, void *arg) { +#ifdef INVARIANTS + uma_zone_t zone = m_getzone(size); - KASSERT(*(uma_find_refcnt(zone_clust, mem)) <= 1, + KASSERT(*(uma_find_refcnt(zone, mem)) <= 1, ("%s: refcnt incorrect %u", __func__, - *(uma_find_refcnt(zone_clust, mem))) ); -#ifdef INVARIANTS + *(uma_find_refcnt(zone, mem))) ); + trash_dtor(mem, size, arg); #endif } ==== //depot/projects/opentoe/sys/sys/mbuf.h#3 (text+ko) ==== @@ -185,6 +185,8 @@ #define M_LASTFRAG 0x2000 /* packet is last fragment */ #define M_VLANTAG 0x10000 /* ether_vtag is valid */ #define M_PROMISC 0x20000 /* packet was not for us */ +#define M_IOVEC 0x40000 /* mbuf immediate data area is used to reference clusters */ +#define M_LRO 0x80000 /* large receive offload in use for packet */ /* * External buffer types: identify ext_buf type. @@ -275,6 +277,21 @@ }; /* + * m_pktdat == 200 bytes on 64-bit arches, need to stay below that + * + */ +#define MAX_MBUF_IOV 12 +struct mbuf_iovec { + uint16_t mi_first; /* first valid cluster */ + uint16_t mi_count; /* number of valid clusters */ + uint16_t mi_flags[MAX_MBUF_IOV]; /* per-cluster flags */ + uint16_t mi_offsets[MAX_MBUF_IOV];/* data offsets of clusters */ + uint16_t mi_lens[MAX_MBUF_IOV]; /* length of clusters */ + uint32_t pad; /* 8-byte align mi_bases */ + caddr_t mi_bases[MAX_MBUF_IOV]; /* pointers to clusters */ +}; + +/* * Flags specifying how an allocation should be made. * * The flag to use is as follows: @@ -392,6 +409,33 @@ return ((struct mbuf *)(uma_zalloc_arg(zone_pack, &args, how))); } +static __inline uma_zone_t +m_getzone(int size) +{ + uma_zone_t zone; + + switch (size) { + case MCLBYTES: + zone = zone_clust; + break; +#if MJUMPAGESIZE != MCLBYTES + case MJUMPAGESIZE: + zone = zone_jumbop; + break; +#endif + case MJUM9BYTES: + zone = zone_jumbo9; + break; + case MJUM16BYTES: + zone = zone_jumbo16; + break; + default: + panic("%s: m_getjcl: invalid cluster type", __func__); + } + + return (zone); +} + /* * m_getjcl() returns an mbuf with a cluster of the specified size attached. * For size it takes MCLBYTES, MJUMPAGESIZE, MJUM9BYTES, MJUM16BYTES. @@ -412,24 +456,7 @@ if (m == NULL) return (NULL); - switch (size) { - case MCLBYTES: - zone = zone_clust; - break; -#if MJUMPAGESIZE != MCLBYTES - case MJUMPAGESIZE: - zone = zone_jumbop; - break; -#endif - case MJUM9BYTES: - zone = zone_jumbo9; - break; - case MJUM16BYTES: - zone = zone_jumbo16; - break; - default: - panic("%s: m_getjcl: invalid cluster type", __func__); - } + zone = m_getzone(size); n = uma_zalloc_arg(zone, m, how); if (n == NULL) { uma_zfree(zone_mbuf, m); @@ -468,6 +495,7 @@ } } + /* * m_cljget() is different from m_clget() as it can allocate clusters without * attaching them to an mbuf. In that case the return value is the pointer @@ -478,35 +506,55 @@ static __inline void * m_cljget(struct mbuf *m, int how, int size) { - uma_zone_t zone; - + uma_zone_t zone = m_getzone(size); + if (m && m->m_flags & M_EXT) printf("%s: %p mbuf already has cluster\n", __func__, m); if (m != NULL) m->m_ext.ext_buf = NULL; + return (uma_zalloc_arg(zone, m, how)); +} + +static __inline void +m_cljset(struct mbuf *m, void *cl, int size) +{ + uma_zone_t zone; + int type; + switch (size) { case MCLBYTES: + type = EXT_CLUSTER; zone = zone_clust; break; #if MJUMPAGESIZE != MCLBYTES case MJUMPAGESIZE: + type = EXT_JUMBOP; zone = zone_jumbop; break; #endif case MJUM9BYTES: + type = EXT_JUMBO9; zone = zone_jumbo9; break; case MJUM16BYTES: + type = EXT_JUMBO16; zone = zone_jumbo16; break; default: - panic("%s: m_getjcl: invalid cluster type", __func__); + panic("unknown cluster size"); + break; } - return (uma_zalloc_arg(zone, m, how)); + m->m_data = m->m_ext.ext_buf = cl; + m->m_ext.ext_free = m->m_ext.ext_args = NULL; + m->m_ext.ext_size = size; + m->m_ext.ext_type = type; + m->m_ext.ref_cnt = uma_find_refcnt(zone, cl); + m->m_flags |= M_EXT; + } - + static __inline void m_chtype(struct mbuf *m, short new_type) {