Skip site navigation (1)Skip section navigation (2)
Date:      Sat, 2 May 2020 22:39:27 +0000 (UTC)
From:      Gleb Smirnoff <glebius@FreeBSD.org>
To:        src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org
Subject:   svn commit: r360569 - in head/sys: dev/cxgbe dev/cxgbe/crypto dev/cxgbe/tom kern sys
Message-ID:  <202005022239.042MdR7Z036054@repo.freebsd.org>

next in thread | raw e-mail | index | archive | help
Author: glebius
Date: Sat May  2 22:39:26 2020
New Revision: 360569
URL: https://svnweb.freebsd.org/changeset/base/360569

Log:
  Continuation of multi page mbuf redesign from r359919.
  
  The following series of patches addresses three things:
  
  Now that array of pages is embedded into mbuf, we no longer need
  separate structure to pass around, so struct mbuf_ext_pgs is an
  artifact of the first implementation. And struct mbuf_ext_pgs_data
  is a crutch to accomodate the main idea r359919 with minimal churn.
  
  Also, M_EXT of type EXT_PGS are just a synonym of M_NOMAP.
  
  The namespace for the newfeature is somewhat inconsistent and
  sometimes has a lengthy prefixes. In these patches we will
  gradually bring the namespace to "m_epg" prefix for all mbuf
  fields and most functions.
  
  Step 1 of 4:
  
   o Anonymize mbuf_ext_pgs_data, embed in m_ext
   o Embed mbuf_ext_pgs
   o Start documenting all this entanglement
  
  Reviewed by:	gallatin
  Differential Revision:	https://reviews.freebsd.org/D24598

Modified:
  head/sys/dev/cxgbe/crypto/t4_kern_tls.c
  head/sys/dev/cxgbe/t4_sge.c
  head/sys/dev/cxgbe/tom/t4_cpl_io.c
  head/sys/dev/cxgbe/tom/t4_tls.c
  head/sys/kern/kern_mbuf.c
  head/sys/kern/kern_sendfile.c
  head/sys/kern/subr_bus_dma.c
  head/sys/kern/subr_sglist.c
  head/sys/kern/uipc_ktls.c
  head/sys/kern/uipc_mbuf.c
  head/sys/sys/mbuf.h
  head/sys/sys/sglist.h

Modified: head/sys/dev/cxgbe/crypto/t4_kern_tls.c
==============================================================================
--- head/sys/dev/cxgbe/crypto/t4_kern_tls.c	Sat May  2 20:47:58 2020	(r360568)
+++ head/sys/dev/cxgbe/crypto/t4_kern_tls.c	Sat May  2 22:39:26 2020	(r360569)
@@ -906,7 +906,7 @@ ktls_tcp_payload_length(struct tlspcb *tlsp, struct mb
 
 	MBUF_EXT_PGS_ASSERT(m_tls);
 	ext_pgs = &m_tls->m_ext_pgs;
-	hdr = (void *)ext_pgs->m_epg_hdr;
+	hdr = (void *)m_tls->m_epg_hdr;
 	plen = ntohs(hdr->tls_length);
 
 	/*
@@ -962,7 +962,7 @@ ktls_payload_offset(struct tlspcb *tlsp, struct mbuf *
 
 	MBUF_EXT_PGS_ASSERT(m_tls);
 	ext_pgs = &m_tls->m_ext_pgs;
-	hdr = (void *)ext_pgs->m_epg_hdr;
+	hdr = (void *)m_tls->m_epg_hdr;
 	plen = ntohs(hdr->tls_length);
 #ifdef INVARIANTS
 	mlen = mtod(m_tls, vm_offset_t) + m_tls->m_len;
@@ -1040,7 +1040,7 @@ ktls_wr_len(struct tlspcb *tlsp, struct mbuf *m, struc
 		return (wr_len);
 	}
 
-	hdr = (void *)ext_pgs->m_epg_hdr;
+	hdr = (void *)m_tls->m_epg_hdr;
 	plen = TLS_HEADER_LENGTH + ntohs(hdr->tls_length) - ext_pgs->trail_len;
 	if (tlen < plen) {
 		plen = tlen;
@@ -1064,7 +1064,7 @@ ktls_wr_len(struct tlspcb *tlsp, struct mbuf *m, struc
 	wr_len += roundup2(imm_len, 16);
 
 	/* TLS record payload via DSGL. */
-	*nsegsp = sglist_count_ext_pgs(ext_pgs, ext_pgs->hdr_len + offset,
+	*nsegsp = sglist_count_ext_pgs(m_tls, ext_pgs->hdr_len + offset,
 	    plen - (ext_pgs->hdr_len + offset));
 	wr_len += ktls_sgl_size(*nsegsp);
 
@@ -1543,7 +1543,7 @@ ktls_write_tunnel_packet(struct sge_txq *txq, void *ds
 	    (m->m_pkthdr.l2hlen + m->m_pkthdr.l3hlen + sizeof(*tcp)));
 
 	/* Copy the subset of the TLS header requested. */
-	copy_to_txd(&txq->eq, (char *)ext_pgs->m_epg_hdr +
+	copy_to_txd(&txq->eq, (char *)m_tls->m_epg_hdr +
 	    mtod(m_tls, vm_offset_t), &out, m_tls->m_len);
 	txq->imm_wrs++;
 
@@ -1604,7 +1604,7 @@ ktls_write_tls_wr(struct tlspcb *tlsp, struct sge_txq 
 	/* Locate the TLS header. */
 	MBUF_EXT_PGS_ASSERT(m_tls);
 	ext_pgs = &m_tls->m_ext_pgs;
-	hdr = (void *)ext_pgs->m_epg_hdr;
+	hdr = (void *)m_tls->m_epg_hdr;
 	plen = TLS_HEADER_LENGTH + ntohs(hdr->tls_length) - ext_pgs->trail_len;
 
 	/* Determine how much of the TLS record to send. */
@@ -1799,7 +1799,7 @@ ktls_write_tls_wr(struct tlspcb *tlsp, struct sge_txq 
 
 	/* Recalculate 'nsegs' if cached value is not available. */
 	if (nsegs == 0)
-		nsegs = sglist_count_ext_pgs(ext_pgs, ext_pgs->hdr_len +
+		nsegs = sglist_count_ext_pgs(m_tls, ext_pgs->hdr_len +
 		    offset, plen - (ext_pgs->hdr_len + offset));
 
 	/* Calculate the size of the TLS work request. */
@@ -2031,7 +2031,7 @@ ktls_write_tls_wr(struct tlspcb *tlsp, struct sge_txq 
 	/* Populate the TLS header */
 	out = (void *)(tx_data + 1);
 	if (offset == 0) {
-		memcpy(out, ext_pgs->m_epg_hdr, ext_pgs->hdr_len);
+		memcpy(out, m_tls->m_epg_hdr, ext_pgs->hdr_len);
 		out += ext_pgs->hdr_len;
 	}
 
@@ -2067,7 +2067,7 @@ ktls_write_tls_wr(struct tlspcb *tlsp, struct sge_txq 
 
 	/* SGL for record payload */
 	sglist_reset(txq->gl);
-	if (sglist_append_ext_pgs(txq->gl, ext_pgs, ext_pgs->hdr_len + offset,
+	if (sglist_append_ext_pgs(txq->gl, m_tls, ext_pgs->hdr_len + offset,
 	    plen - (ext_pgs->hdr_len + offset)) != 0) {
 #ifdef INVARIANTS
 		panic("%s: failed to append sglist", __func__);

Modified: head/sys/dev/cxgbe/t4_sge.c
==============================================================================
--- head/sys/dev/cxgbe/t4_sge.c	Sat May  2 20:47:58 2020	(r360568)
+++ head/sys/dev/cxgbe/t4_sge.c	Sat May  2 22:39:26 2020	(r360569)
@@ -2435,7 +2435,7 @@ count_mbuf_ext_pgs(struct mbuf *m, int skip, vm_paddr_
 			off = 0;
 			len -= seglen;
 			paddr = pmap_kextract(
-			    (vm_offset_t)&ext_pgs->m_epg_hdr[segoff]);
+			    (vm_offset_t)&m->m_epg_hdr[segoff]);
 			if (*nextaddr != paddr)
 				nsegs++;
 			*nextaddr = paddr + seglen;
@@ -2454,7 +2454,7 @@ count_mbuf_ext_pgs(struct mbuf *m, int skip, vm_paddr_
 		off = 0;
 		seglen = min(seglen, len);
 		len -= seglen;
-		paddr = ext_pgs->m_epg_pa[i] + segoff;
+		paddr = m->m_epg_pa[i] + segoff;
 		if (*nextaddr != paddr)
 			nsegs++;
 		*nextaddr = paddr + seglen;
@@ -2463,7 +2463,7 @@ count_mbuf_ext_pgs(struct mbuf *m, int skip, vm_paddr_
 	if (len != 0) {
 		seglen = min(len, ext_pgs->trail_len - off);
 		len -= seglen;
-		paddr = pmap_kextract((vm_offset_t)&ext_pgs->m_epg_trail[off]);
+		paddr = pmap_kextract((vm_offset_t)&m->m_epg_trail[off]);
 		if (*nextaddr != paddr)
 			nsegs++;
 		*nextaddr = paddr + seglen;

Modified: head/sys/dev/cxgbe/tom/t4_cpl_io.c
==============================================================================
--- head/sys/dev/cxgbe/tom/t4_cpl_io.c	Sat May  2 20:47:58 2020	(r360568)
+++ head/sys/dev/cxgbe/tom/t4_cpl_io.c	Sat May  2 22:39:26 2020	(r360569)
@@ -1935,7 +1935,7 @@ aiotx_free_pgs(struct mbuf *m)
 #endif
 
 	for (int i = 0; i < ext_pgs->npgs; i++) {
-		pg = PHYS_TO_VM_PAGE(ext_pgs->m_epg_pa[i]);
+		pg = PHYS_TO_VM_PAGE(m->m_epg_pa[i]);
 		vm_page_unwire(pg, PQ_ACTIVE);
 	}
 
@@ -2003,7 +2003,7 @@ alloc_aiotx_mbuf(struct kaiocb *job, int len)
 			    (npages - 2) * PAGE_SIZE;
 		}
 		for (i = 0; i < npages; i++)
-			ext_pgs->m_epg_pa[i] = VM_PAGE_TO_PHYS(pgs[i]);
+			m->m_epg_pa[i] = VM_PAGE_TO_PHYS(pgs[i]);
 
 		m->m_len = mlen;
 		m->m_ext.ext_size = npages * PAGE_SIZE;

Modified: head/sys/dev/cxgbe/tom/t4_tls.c
==============================================================================
--- head/sys/dev/cxgbe/tom/t4_tls.c	Sat May  2 20:47:58 2020	(r360568)
+++ head/sys/dev/cxgbe/tom/t4_tls.c	Sat May  2 22:39:26 2020	(r360569)
@@ -1623,26 +1623,24 @@ t4_push_tls_records(struct adapter *sc, struct toepcb 
 
 #ifdef KERN_TLS
 static int
-count_ext_pgs_segs(struct mbuf_ext_pgs *ext_pgs,
-	struct mbuf_ext_pgs_data *ext_pgs_data)
+count_ext_pgs_segs(struct mbuf *m)
 {
 	vm_paddr_t nextpa;
 	u_int i, nsegs;
 
-	MPASS(ext_pgs->npgs > 0);
+	MPASS(m->m_ext_pgs.npgs > 0);
 	nsegs = 1;
-	nextpa = ext_pgs_data->pa[0] + PAGE_SIZE;
-	for (i = 1; i < ext_pgs->npgs; i++) {
-		if (nextpa != ext_pgs_data->pa[i])
+	nextpa = m->m_epg_pa[0] + PAGE_SIZE;
+	for (i = 1; i < m->m_ext_pgs.npgs; i++) {
+		if (nextpa != m->m_epg_pa[i])
 			nsegs++;
-		nextpa = ext_pgs_data->pa[i] + PAGE_SIZE;
+		nextpa = m->m_epg_pa[i] + PAGE_SIZE;
 	}
 	return (nsegs);
 }
 
 static void
-write_ktlstx_sgl(void *dst, struct mbuf_ext_pgs *ext_pgs,
-    struct mbuf_ext_pgs_data *ext_pgs_data, int nsegs)
+write_ktlstx_sgl(void *dst, struct mbuf *m, int nsegs)
 {
 	struct ulptx_sgl *usgl = dst;
 	vm_paddr_t pa;
@@ -1655,15 +1653,15 @@ write_ktlstx_sgl(void *dst, struct mbuf_ext_pgs *ext_p
 	    V_ULPTX_NSGE(nsegs));
 
 	/* Figure out the first S/G length. */
-	pa = ext_pgs_data->pa[0] + ext_pgs->first_pg_off;
+	pa = m->m_epg_pa[0] + m->m_ext_pgs.first_pg_off;
 	usgl->addr0 = htobe64(pa);
-	len = mbuf_ext_pg_len(ext_pgs, 0, ext_pgs->first_pg_off);
+	len = mbuf_ext_pg_len(&m->m_ext_pgs, 0, m->m_ext_pgs.first_pg_off);
 	pa += len;
-	for (i = 1; i < ext_pgs->npgs; i++) {
-		if (ext_pgs_data->pa[i] != pa)
+	for (i = 1; i < m->m_ext_pgs.npgs; i++) {
+		if (m->m_epg_pa[i] != pa)
 			break;
-		len += mbuf_ext_pg_len(ext_pgs, i, 0);
-		pa += mbuf_ext_pg_len(ext_pgs, i, 0);
+		len += mbuf_ext_pg_len(&m->m_ext_pgs, i, 0);
+		pa += mbuf_ext_pg_len(&m->m_ext_pgs, i, 0);
 	}
 	usgl->len0 = htobe32(len);
 #ifdef INVARIANTS
@@ -1671,21 +1669,21 @@ write_ktlstx_sgl(void *dst, struct mbuf_ext_pgs *ext_p
 #endif
 
 	j = -1;
-	for (; i < ext_pgs->npgs; i++) {
-		if (j == -1 || ext_pgs_data->pa[i] != pa) {
+	for (; i < m->m_ext_pgs.npgs; i++) {
+		if (j == -1 || m->m_epg_pa[i] != pa) {
 			if (j >= 0)
 				usgl->sge[j / 2].len[j & 1] = htobe32(len);
 			j++;
 #ifdef INVARIANTS
 			nsegs--;
 #endif
-			pa = ext_pgs_data->pa[i];
+			pa = m->m_epg_pa[i];
 			usgl->sge[j / 2].addr[j & 1] = htobe64(pa);
-			len = mbuf_ext_pg_len(ext_pgs, i, 0);
+			len = mbuf_ext_pg_len(&m->m_ext_pgs, i, 0);
 			pa += len;
 		} else {
-			len += mbuf_ext_pg_len(ext_pgs, i, 0);
-			pa += mbuf_ext_pg_len(ext_pgs, i, 0);
+			len += mbuf_ext_pg_len(&m->m_ext_pgs, i, 0);
+			pa += mbuf_ext_pg_len(&m->m_ext_pgs, i, 0);
 		}
 	}
 	if (j >= 0) {
@@ -1694,8 +1692,7 @@ write_ktlstx_sgl(void *dst, struct mbuf_ext_pgs *ext_p
 		if ((j & 1) == 0)
 			usgl->sge[j / 2].len[1] = htobe32(0);
 	}
-	KASSERT(nsegs == 0, ("%s: nsegs %d, ext_pgs %p", __func__, nsegs,
-	    ext_pgs));
+	KASSERT(nsegs == 0, ("%s: nsegs %d, m %p", __func__, nsegs, m));
 }
 
 /*
@@ -1813,8 +1810,7 @@ t4_push_ktls(struct adapter *sc, struct toepcb *toep, 
 		wr_len += AES_BLOCK_LEN;
 
 		/* Account for SGL in work request length. */
-		nsegs = count_ext_pgs_segs(&m->m_ext_pgs,
-		    &m->m_ext.ext_pgs);
+		nsegs = count_ext_pgs_segs(m);
 		wr_len += sizeof(struct ulptx_sgl) +
 		    ((3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1)) * 8;
 
@@ -1892,8 +1888,7 @@ t4_push_ktls(struct adapter *sc, struct toepcb *toep, 
 		memcpy(buf, thdr + 1, toep->tls.iv_len);
 		buf += AES_BLOCK_LEN;
 
-		write_ktlstx_sgl(buf, &m->m_ext_pgs, &m->m_ext.ext_pgs,
-		    nsegs);
+		write_ktlstx_sgl(buf, m, nsegs);
 
 		KASSERT(toep->tx_credits >= credits,
 			("%s: not enough credits", __func__));

Modified: head/sys/kern/kern_mbuf.c
==============================================================================
--- head/sys/kern/kern_mbuf.c	Sat May  2 20:47:58 2020	(r360568)
+++ head/sys/kern/kern_mbuf.c	Sat May  2 22:39:26 2020	(r360569)
@@ -311,9 +311,6 @@ static void	mb_reclaim(uma_zone_t, int);
 /* Ensure that MSIZE is a power of 2. */
 CTASSERT((((MSIZE - 1) ^ MSIZE) + 1) >> 1 == MSIZE);
 
-_Static_assert(offsetof(struct mbuf, m_ext) ==
-    offsetof(struct mbuf, m_ext_pgs.m_ext),
-    "m_ext offset mismatch between mbuf and ext_pgs");
 _Static_assert(sizeof(struct mbuf) <= MSIZE,
     "size of mbuf exceeds MSIZE");
 /*
@@ -984,7 +981,7 @@ _mb_unmapped_to_ext(struct mbuf *m)
 				goto fail;
 			m_new->m_len = seglen;
 			prev = top = m_new;
-			memcpy(mtod(m_new, void *), &ext_pgs->m_epg_hdr[segoff],
+			memcpy(mtod(m_new, void *), &m->m_epg_hdr[segoff],
 			    seglen);
 		}
 	}
@@ -1002,7 +999,7 @@ _mb_unmapped_to_ext(struct mbuf *m)
 		seglen = min(seglen, len);
 		len -= seglen;
 
-		pg = PHYS_TO_VM_PAGE(ext_pgs->m_epg_pa[i]);
+		pg = PHYS_TO_VM_PAGE(m->m_epg_pa[i]);
 		m_new = m_get(M_NOWAIT, MT_DATA);
 		if (m_new == NULL)
 			goto fail;
@@ -1036,7 +1033,7 @@ _mb_unmapped_to_ext(struct mbuf *m)
 		else
 			prev->m_next = m_new;
 		m_new->m_len = len;
-		memcpy(mtod(m_new, void *), &ext_pgs->m_epg_trail[off], len);
+		memcpy(mtod(m_new, void *), &m->m_epg_trail[off], len);
 	}
 
 	if (ref_inc != 0) {
@@ -1154,8 +1151,9 @@ mb_alloc_ext_pgs(int how, m_ext_free_t ext_free)
 
 #ifdef INVARIANT_SUPPORT
 void
-mb_ext_pgs_check(struct mbuf_ext_pgs *ext_pgs)
+mb_ext_pgs_check(struct mbuf *m)
 {
+	struct mbuf_ext_pgs *ext_pgs = &m->m_ext_pgs;
 
 	/*
 	 * NB: This expects a non-empty buffer (npgs > 0 and
@@ -1163,7 +1161,7 @@ mb_ext_pgs_check(struct mbuf_ext_pgs *ext_pgs)
 	 */
 	KASSERT(ext_pgs->npgs > 0,
 	    ("ext_pgs with no valid pages: %p", ext_pgs));
-	KASSERT(ext_pgs->npgs <= nitems(ext_pgs->m_epg_pa),
+	KASSERT(ext_pgs->npgs <= nitems(m->m_epg_pa),
 	    ("ext_pgs with too many pages: %p", ext_pgs));
 	KASSERT(ext_pgs->nrdy <= ext_pgs->npgs,
 	    ("ext_pgs with too many ready pages: %p", ext_pgs));
@@ -1178,9 +1176,9 @@ mb_ext_pgs_check(struct mbuf_ext_pgs *ext_pgs)
 		    PAGE_SIZE, ("ext_pgs with single page too large: %p",
 		    ext_pgs));
 	}
-	KASSERT(ext_pgs->hdr_len <= sizeof(ext_pgs->m_epg_hdr),
+	KASSERT(ext_pgs->hdr_len <= sizeof(m->m_epg_hdr),
 	    ("ext_pgs with too large header length: %p", ext_pgs));
-	KASSERT(ext_pgs->trail_len <= sizeof(ext_pgs->m_epg_trail),
+	KASSERT(ext_pgs->trail_len <= sizeof(m->m_epg_trail),
 	    ("ext_pgs with too large header length: %p", ext_pgs));
 }
 #endif

Modified: head/sys/kern/kern_sendfile.c
==============================================================================
--- head/sys/kern/kern_sendfile.c	Sat May  2 20:47:58 2020	(r360568)
+++ head/sys/kern/kern_sendfile.c	Sat May  2 22:39:26 2020	(r360569)
@@ -203,7 +203,7 @@ sendfile_free_mext_pg(struct mbuf *m)
 	for (i = 0; i < ext_pgs->npgs; i++) {
 		if (cache_last && i == ext_pgs->npgs - 1)
 			flags = 0;
-		pg = PHYS_TO_VM_PAGE(ext_pgs->m_epg_pa[i]);
+		pg = PHYS_TO_VM_PAGE(m->m_epg_pa[i]);
 		vm_page_release(pg, flags);
 	}
 
@@ -1046,11 +1046,11 @@ retry_space:
 					ext_pgs->nrdy++;
 				}
 
-				ext_pgs->m_epg_pa[ext_pgs_idx] = VM_PAGE_TO_PHYS(pga);
+				m0->m_epg_pa[ext_pgs_idx] = VM_PAGE_TO_PHYS(pga);
 				ext_pgs->npgs++;
 				xfs = xfsize(i, npages, off, space);
 				ext_pgs->last_pg_len = xfs;
-				MBUF_EXT_PGS_ASSERT_SANITY(ext_pgs);
+				MBUF_EXT_PGS_ASSERT_SANITY(m0);
 				mtail->m_len += xfs;
 				mtail->m_ext.ext_size += PAGE_SIZE;
 				continue;

Modified: head/sys/kern/subr_bus_dma.c
==============================================================================
--- head/sys/kern/subr_bus_dma.c	Sat May  2 20:47:58 2020	(r360568)
+++ head/sys/kern/subr_bus_dma.c	Sat May  2 22:39:26 2020	(r360569)
@@ -141,7 +141,7 @@ _bus_dmamap_load_unmapped_mbuf_sg(bus_dma_tag_t dmat, 
 			off = 0;
 			len -= seglen;
 			error = _bus_dmamap_load_buffer(dmat, map,
-			    &ext_pgs->m_epg_hdr[segoff], seglen, kernel_pmap,
+			    &m->m_epg_hdr[segoff], seglen, kernel_pmap,
 			    flags, segs, nsegs);
 		}
 	}
@@ -159,7 +159,7 @@ _bus_dmamap_load_unmapped_mbuf_sg(bus_dma_tag_t dmat, 
 		seglen = min(seglen, len);
 		len -= seglen;
 		error = _bus_dmamap_load_phys(dmat, map,
-		    ext_pgs->m_epg_pa[i] + segoff, seglen, flags, segs, nsegs);
+		    m->m_epg_pa[i] + segoff, seglen, flags, segs, nsegs);
 		pgoff = 0;
 	};
 	if (len != 0 && error == 0) {
@@ -167,7 +167,7 @@ _bus_dmamap_load_unmapped_mbuf_sg(bus_dma_tag_t dmat, 
 		    ("off + len > trail (%d + %d > %d)", off, len,
 		    ext_pgs->trail_len));
 		error = _bus_dmamap_load_buffer(dmat, map,
-		    &ext_pgs->m_epg_trail[off], len, kernel_pmap, flags, segs,
+		    &m->m_epg_trail[off], len, kernel_pmap, flags, segs,
 		    nsegs);
 	}
 	return (error);

Modified: head/sys/kern/subr_sglist.c
==============================================================================
--- head/sys/kern/subr_sglist.c	Sat May  2 20:47:58 2020	(r360568)
+++ head/sys/kern/subr_sglist.c	Sat May  2 22:39:26 2020	(r360569)
@@ -223,8 +223,9 @@ sglist_count_vmpages(vm_page_t *m, size_t pgoff, size_
  * describe an EXT_PGS buffer.
  */
 int
-sglist_count_ext_pgs(struct mbuf_ext_pgs *ext_pgs, size_t off, size_t len)
+sglist_count_ext_pgs(struct mbuf *m, size_t off, size_t len)
 {
+	struct mbuf_ext_pgs *ext_pgs = &m->m_ext_pgs;
 	vm_paddr_t nextaddr, paddr;
 	size_t seglen, segoff;
 	int i, nsegs, pglen, pgoff;
@@ -242,7 +243,7 @@ sglist_count_ext_pgs(struct mbuf_ext_pgs *ext_pgs, siz
 			seglen = MIN(seglen, len);
 			off = 0;
 			len -= seglen;
-			nsegs += sglist_count(&ext_pgs->m_epg_hdr[segoff],
+			nsegs += sglist_count(&m->m_epg_hdr[segoff],
 			    seglen);
 		}
 	}
@@ -260,7 +261,7 @@ sglist_count_ext_pgs(struct mbuf_ext_pgs *ext_pgs, siz
 		off = 0;
 		seglen = MIN(seglen, len);
 		len -= seglen;
-		paddr = ext_pgs->m_epg_pa[i] + segoff;
+		paddr = m->m_epg_pa[i] + segoff;
 		if (paddr != nextaddr)
 			nsegs++;
 		nextaddr = paddr + seglen;
@@ -269,7 +270,7 @@ sglist_count_ext_pgs(struct mbuf_ext_pgs *ext_pgs, siz
 	if (len != 0) {
 		seglen = MIN(len, ext_pgs->trail_len - off);
 		len -= seglen;
-		nsegs += sglist_count(&ext_pgs->m_epg_trail[off], seglen);
+		nsegs += sglist_count(&m->m_epg_trail[off], seglen);
 	}
 	KASSERT(len == 0, ("len != 0"));
 	return (nsegs);
@@ -284,8 +285,7 @@ sglist_count_mb_ext_pgs(struct mbuf *m)
 {
 
 	MBUF_EXT_PGS_ASSERT(m);
-	return (sglist_count_ext_pgs(&m->m_ext_pgs, mtod(m, vm_offset_t),
-	    m->m_len));
+	return (sglist_count_ext_pgs(m, mtod(m, vm_offset_t), m->m_len));
 }
 
 /*
@@ -395,9 +395,9 @@ sglist_append_phys(struct sglist *sg, vm_paddr_t paddr
  * fails with EFBIG.
  */
 int
-sglist_append_ext_pgs(struct sglist *sg, struct mbuf_ext_pgs *ext_pgs,
-    size_t off, size_t len)
+sglist_append_ext_pgs(struct sglist *sg, struct mbuf *m, size_t off, size_t len)
 {
+	struct mbuf_ext_pgs *ext_pgs = &m->m_ext_pgs;
 	size_t seglen, segoff;
 	vm_paddr_t paddr;
 	int error, i, pglen, pgoff;
@@ -413,7 +413,7 @@ sglist_append_ext_pgs(struct sglist *sg, struct mbuf_e
 			off = 0;
 			len -= seglen;
 			error = sglist_append(sg,
-			    &ext_pgs->m_epg_hdr[segoff], seglen);
+			    &m->m_epg_hdr[segoff], seglen);
 		}
 	}
 	pgoff = ext_pgs->first_pg_off;
@@ -429,7 +429,7 @@ sglist_append_ext_pgs(struct sglist *sg, struct mbuf_e
 		off = 0;
 		seglen = MIN(seglen, len);
 		len -= seglen;
-		paddr = ext_pgs->m_epg_pa[i] + segoff;
+		paddr = m->m_epg_pa[i] + segoff;
 		error = sglist_append_phys(sg, paddr, seglen);
 		pgoff = 0;
 	};
@@ -437,7 +437,7 @@ sglist_append_ext_pgs(struct sglist *sg, struct mbuf_e
 		seglen = MIN(len, ext_pgs->trail_len - off);
 		len -= seglen;
 		error = sglist_append(sg,
-		    &ext_pgs->m_epg_trail[off], seglen);
+		    &m->m_epg_trail[off], seglen);
 	}
 	if (error == 0)
 		KASSERT(len == 0, ("len != 0"));
@@ -455,8 +455,7 @@ sglist_append_mb_ext_pgs(struct sglist *sg, struct mbu
 
 	/* for now, all unmapped mbufs are assumed to be EXT_PGS */
 	MBUF_EXT_PGS_ASSERT(m);
-	return (sglist_append_ext_pgs(sg, &m->m_ext_pgs,
-	    mtod(m, vm_offset_t), m->m_len));
+	return (sglist_append_ext_pgs(sg, m, mtod(m, vm_offset_t), m->m_len));
 }
 
 /*

Modified: head/sys/kern/uipc_ktls.c
==============================================================================
--- head/sys/kern/uipc_ktls.c	Sat May  2 20:47:58 2020	(r360568)
+++ head/sys/kern/uipc_ktls.c	Sat May  2 22:39:26 2020	(r360569)
@@ -1374,7 +1374,7 @@ ktls_frame(struct mbuf *top, struct ktls_session *tls,
 		m->m_len += pgs->hdr_len + pgs->trail_len;
 
 		/* Populate the TLS header. */
-		tlshdr = (void *)pgs->m_epg_hdr;
+		tlshdr = (void *)m->m_epg_hdr;
 		tlshdr->tls_vmajor = tls->params.tls_vmajor;
 
 		/*
@@ -1387,7 +1387,7 @@ ktls_frame(struct mbuf *top, struct ktls_session *tls,
 			tlshdr->tls_type = TLS_RLTYPE_APP;
 			/* save the real record type for later */
 			pgs->record_type = record_type;
-			pgs->m_epg_trail[0] = record_type;
+			m->m_epg_trail[0] = record_type;
 		} else {
 			tlshdr->tls_vminor = tls->params.tls_vminor;
 			tlshdr->tls_type = record_type;
@@ -1552,7 +1552,7 @@ ktls_encrypt(struct mbuf_ext_pgs *pgs)
 			len = mbuf_ext_pg_len(pgs, i, off);
 			src_iov[i].iov_len = len;
 			src_iov[i].iov_base =
-			    (char *)(void *)PHYS_TO_DMAP(pgs->m_epg_pa[i]) +
+			    (char *)(void *)PHYS_TO_DMAP(m->m_epg_pa[i]) +
 				off;
 
 			if (is_anon) {
@@ -1576,8 +1576,8 @@ retry_page:
 		npages += i;
 
 		error = (*tls->sw_encrypt)(tls,
-		    (const struct tls_record_layer *)pgs->m_epg_hdr,
-		    pgs->m_epg_trail, src_iov, dst_iov, i, pgs->seqno,
+		    (const struct tls_record_layer *)m->m_epg_hdr,
+		    m->m_epg_trail, src_iov, dst_iov, i, pgs->seqno,
 		    pgs->record_type);
 		if (error) {
 			counter_u64_add(ktls_offload_failed_crypto, 1);
@@ -1595,7 +1595,7 @@ retry_page:
 
 			/* Replace them with the new pages. */
 			for (i = 0; i < pgs->npgs; i++)
-				pgs->m_epg_pa[i] = parray[i];
+				m->m_epg_pa[i] = parray[i];
 
 			/* Use the basic free routine. */
 			m->m_ext.ext_free = mb_free_mext_pgs;

Modified: head/sys/kern/uipc_mbuf.c
==============================================================================
--- head/sys/kern/uipc_mbuf.c	Sat May  2 20:47:58 2020	(r360568)
+++ head/sys/kern/uipc_mbuf.c	Sat May  2 22:39:26 2020	(r360569)
@@ -163,11 +163,11 @@ CTASSERT(offsetof(struct mbuf, m_pktdat) % 8 == 0);
 #if defined(__LP64__)
 CTASSERT(offsetof(struct mbuf, m_dat) == 32);
 CTASSERT(sizeof(struct pkthdr) == 56);
-CTASSERT(sizeof(struct m_ext) == 168);
+CTASSERT(sizeof(struct m_ext) == 160);
 #else
 CTASSERT(offsetof(struct mbuf, m_dat) == 24);
 CTASSERT(sizeof(struct pkthdr) == 48);
-CTASSERT(sizeof(struct m_ext) == 184);
+CTASSERT(sizeof(struct m_ext) == 180);
 #endif
 
 /*
@@ -195,19 +195,30 @@ mb_dupcl(struct mbuf *n, struct mbuf *m)
 	KASSERT(!(n->m_flags & M_EXT), ("%s: M_EXT set on %p", __func__, n));
 
 	/*
-	 * Cache access optimization.  For most kinds of external
-	 * storage we don't need full copy of m_ext, since the
-	 * holder of the 'ext_count' is responsible to carry the
-	 * free routine and its arguments.  Exclusion is EXT_EXTREF,
-	 * where 'ext_cnt' doesn't point into mbuf at all.
+	 * Cache access optimization.
+	 *
+	 * o Regular M_EXT storage doesn't need full copy of m_ext, since
+	 *   the holder of the 'ext_count' is responsible to carry the free
+	 *   routine and its arguments.
+	 * o EXT_PGS data is split between main part of mbuf and m_ext, the
+	 *   main part is copied in full, the m_ext part is similar to M_EXT.
+	 * o EXT_EXTREF, where 'ext_cnt' doesn't point into mbuf at all, is
+	 *   special - it needs full copy of m_ext into each mbuf, since any
+	 *   copy could end up as the last to free.
 	 */
-	if (m->m_ext.ext_type == EXT_EXTREF)
-		bcopy(&m->m_ext, &n->m_ext, sizeof(struct m_ext));
-	else if (m->m_ext.ext_type == EXT_PGS)
+	switch (m->m_ext.ext_type) {
+	case EXT_PGS:
+		bcopy(&m->m_ext, &n->m_ext, m_epg_copylen);
 		bcopy(&m->m_ext_pgs, &n->m_ext_pgs,
 		    sizeof(struct mbuf_ext_pgs));
-	else
+		break;
+	case EXT_EXTREF:
+		bcopy(&m->m_ext, &n->m_ext, sizeof(struct m_ext));
+		break;
+	default:
 		bcopy(&m->m_ext, &n->m_ext, m_ext_copylen);
+	}
+
 	n->m_flags |= M_EXT;
 	n->m_flags |= m->m_flags & (M_RDONLY | M_NOMAP);
 
@@ -1623,7 +1634,7 @@ mb_free_mext_pgs(struct mbuf *m)
 	MBUF_EXT_PGS_ASSERT(m);
 	ext_pgs = &m->m_ext_pgs;
 	for (int i = 0; i < ext_pgs->npgs; i++) {
-		pg = PHYS_TO_VM_PAGE(ext_pgs->m_epg_pa[i]);
+		pg = PHYS_TO_VM_PAGE(m->m_epg_pa[i]);
 		vm_page_unwire_noq(pg);
 		vm_page_free(pg);
 	}
@@ -1681,11 +1692,11 @@ retry_page:
 				}
 			}
 			pg_array[i]->flags &= ~PG_ZERO;
-			pgs->m_epg_pa[i] = VM_PAGE_TO_PHYS(pg_array[i]);
+			mb->m_epg_pa[i] = VM_PAGE_TO_PHYS(pg_array[i]);
 			pgs->npgs++;
 		}
 		pgs->last_pg_len = length - PAGE_SIZE * (pgs->npgs - 1);
-		MBUF_EXT_PGS_ASSERT_SANITY(pgs);
+		MBUF_EXT_PGS_ASSERT_SANITY(mb);
 		total -= length;
 		error = uiomove_fromphys(pg_array, 0, length, uio);
 		if (error != 0)
@@ -1788,7 +1799,8 @@ m_unmappedtouio(const struct mbuf *m, int m_off, struc
 			seglen = min(seglen, len);
 			off = 0;
 			len -= seglen;
-			error = uiomove(&ext_pgs->m_epg_hdr[segoff], seglen, uio);
+			error = uiomove(__DECONST(void *,
+			    &m->m_epg_hdr[segoff]), seglen, uio);
 		}
 	}
 	pgoff = ext_pgs->first_pg_off;
@@ -1804,7 +1816,7 @@ m_unmappedtouio(const struct mbuf *m, int m_off, struc
 		off = 0;
 		seglen = min(seglen, len);
 		len -= seglen;
-		pg = PHYS_TO_VM_PAGE(ext_pgs->m_epg_pa[i]);
+		pg = PHYS_TO_VM_PAGE(m->m_epg_pa[i]);
 		error = uiomove_fromphys(&pg, segoff, seglen, uio);
 		pgoff = 0;
 	};
@@ -1812,7 +1824,8 @@ m_unmappedtouio(const struct mbuf *m, int m_off, struc
 		KASSERT((off + len) <= ext_pgs->trail_len,
 		    ("off + len > trail (%d + %d > %d, m_off = %d)", off, len,
 		    ext_pgs->trail_len, m_off));
-		error = uiomove(&ext_pgs->m_epg_trail[off], len, uio);
+		error = uiomove(__DECONST(void *, &m->m_epg_trail[off]),
+		    len, uio);
 	}
 	return (error);
 }

Modified: head/sys/sys/mbuf.h
==============================================================================
--- head/sys/sys/mbuf.h	Sat May  2 20:47:58 2020	(r360568)
+++ head/sys/sys/mbuf.h	Sat May  2 22:39:26 2020	(r360569)
@@ -231,13 +231,6 @@ struct pkthdr {
 
 #define MBUF_PEXT_FLAG_ANON	1	/* Data can be encrypted in place. */
 
-
-struct mbuf_ext_pgs_data {
-	vm_paddr_t	pa[MBUF_PEXT_MAX_PGS];		/* phys addrs of pgs */
-	char		trail[MBUF_PEXT_TRAIL_LEN]; 	/* TLS trailer */
-	char		hdr[MBUF_PEXT_HDR_LEN];		/* TLS header */
-};
-
 struct ktls_session;
 struct socket;
 
@@ -266,49 +259,49 @@ struct m_ext {
 	uint32_t	 ext_size;	/* size of buffer, for ext_free */
 	uint32_t	 ext_type:8,	/* type of external storage */
 			 ext_flags:24;	/* external storage mbuf flags */
-	char		*ext_buf;	/* start of buffer */
-	/*
-	 * Fields below store the free context for the external storage.
-	 * They are valid only in the refcount carrying mbuf, the one with
-	 * EXT_FLAG_EMBREF flag, with exclusion for EXT_EXTREF type, where
-	 * the free context is copied into all mbufs that use same external
-	 * storage.
-	 */
-#define	m_ext_copylen	offsetof(struct m_ext, ext_free)
-	m_ext_free_t	*ext_free;	/* free routine if not the usual */
-	void		*ext_arg1;	/* optional argument pointer */
 	union {
-		void		*ext_arg2;	/* optional argument pointer */
-		struct mbuf_ext_pgs_data ext_pgs;
+		struct {
+			/*
+			 * Regular M_EXT mbuf:
+			 * o ext_buf always points to the external buffer.
+			 * o ext_free (below) and two optional arguments
+			 *   ext_arg1 and ext_arg2 store the free context for
+			 *   the external storage.  They are set only in the
+			 *   refcount carrying mbuf, the one with
+			 *   EXT_FLAG_EMBREF flag, with exclusion for
+			 *   EXT_EXTREF type, where the free context is copied
+			 *   into all mbufs that use same external storage.
+			 */
+			char 	*ext_buf;	/* start of buffer */
+#define	m_ext_copylen	offsetof(struct m_ext, ext_arg2)
+			void	*ext_arg2;
+		};
+		struct {
+			/*
+			 * Multi-page M_EXTPG mbuf:
+			 * o extpg_pa - page vector.
+			 * o extpg_trail and extpg_hdr - TLS trailer and
+			 *   header.
+			 * Uses ext_free and may also use ext_arg1.
+			 */
+			vm_paddr_t	extpg_pa[MBUF_PEXT_MAX_PGS];
+			char		extpg_trail[MBUF_PEXT_TRAIL_LEN];
+			char		extpg_hdr[MBUF_PEXT_HDR_LEN];
+			/* Pretend these 3 fields are part of mbuf itself. */
+#define	m_epg_pa	m_ext.extpg_pa
+#define	m_epg_trail	m_ext.extpg_trail
+#define	m_epg_hdr	m_ext.extpg_hdr
+#define	m_epg_copylen	offsetof(struct m_ext, ext_free)
+		};
 	};
+	/*
+	 * Free method and optional argument pointer, both
+	 * used by M_EXT and M_EXTPG.
+	 */
+	m_ext_free_t	*ext_free;
+	void		*ext_arg1;
 };
 
-struct mbuf_ext_pgs {
-	uint8_t		npgs;			/* Number of attached pages */
-	uint8_t		nrdy;			/* Pages with I/O pending */
-	uint8_t		hdr_len;		/* TLS header length */
-	uint8_t		trail_len;		/* TLS trailer length */
-	uint16_t	first_pg_off;		/* Offset into 1st page */
-	uint16_t	last_pg_len;		/* Length of last page */
-	uint8_t		flags;			/* Flags */
-	uint8_t		record_type;
-	uint8_t		spare[2];
-	int		enc_cnt;
-	struct ktls_session *tls;		/* TLS session */
-	struct socket	*so;
-	uint64_t	seqno;
-	struct mbuf	*mbuf;
-	STAILQ_ENTRY(mbuf_ext_pgs) stailq;
-#if !defined(__LP64__)
-	uint8_t		pad[8];		/* pad to size of pkthdr */
-#endif
-	struct m_ext	m_ext;
-};
-
-#define m_epg_hdr	m_ext.ext_pgs.hdr
-#define m_epg_trail	m_ext.ext_pgs.trail
-#define m_epg_pa	m_ext.ext_pgs.pa
-
 /*
  * The core of the mbuf object along with some shortcut defines for practical
  * purposes.
@@ -347,15 +340,48 @@ struct mbuf {
 	 * order to support future work on variable-size mbufs.
 	 */
 	union {
-		union {
-			struct {
-				struct pkthdr	m_pkthdr; /* M_PKTHDR set */
-				union {
-					struct m_ext	m_ext;	/* M_EXT set */
-					char		m_pktdat[0];
-				};
+		struct {
+			union {
+				/* M_PKTHDR set. */
+				struct pkthdr	m_pkthdr;
+
+				/* M_EXTPG set.
+				 * Multi-page M_EXTPG mbuf has its meta data
+				 * split between the mbuf_ext_pgs structure
+				 * and m_ext.  It carries vector of pages,
+				 * optional header and trailer char vectors
+				 * and pointers to socket/TLS data.
+				 */
+				struct mbuf_ext_pgs {
+					/* Overall count of pages and count of
+					 * pages with I/O pending. */
+					uint8_t	npgs;
+					uint8_t	nrdy;
+					/* TLS header and trailer lengths.
+					 * The data itself resides in m_ext. */
+					uint8_t	hdr_len;
+					uint8_t	trail_len;
+					/* Offset into 1st page and lenght of
+					 * data in the last page. */
+					uint16_t first_pg_off;
+					uint16_t last_pg_len;
+					uint8_t	flags;
+					uint8_t	record_type;
+					uint8_t	spare[2];
+					int	enc_cnt;
+					struct ktls_session *tls;
+					struct socket	*so;
+					uint64_t	seqno;
+					struct mbuf	*mbuf;
+					STAILQ_ENTRY(mbuf_ext_pgs) stailq;
+				} m_ext_pgs;
 			};
-			struct mbuf_ext_pgs m_ext_pgs;
+			union {
+				/* M_EXT or M_EXTPG set. */
+				struct m_ext	m_ext;
+				/* M_PKTHDR set, neither M_EXT nor M_EXTPG. */
+				char		m_pktdat[0];
+			};
 		};
 		char	m_dat[0];			/* !M_PKTHDR, !M_EXT */
 	};
@@ -375,12 +401,12 @@ mbuf_ext_pg_len(struct mbuf_ext_pgs *ext_pgs, int pidx
 }
 
 #ifdef INVARIANT_SUPPORT
-void	mb_ext_pgs_check(struct mbuf_ext_pgs *ext_pgs);
+void	mb_ext_pgs_check(struct mbuf *m);
 #endif
 #ifdef INVARIANTS
-#define	MBUF_EXT_PGS_ASSERT_SANITY(ext_pgs)	mb_ext_pgs_check((ext_pgs))
+#define	MBUF_EXT_PGS_ASSERT_SANITY(m)	mb_ext_pgs_check((m))
 #else
-#define	MBUF_EXT_PGS_ASSERT_SANITY(ext_pgs)
+#define	MBUF_EXT_PGS_ASSERT_SANITY(m)
 #endif
 #endif
 

Modified: head/sys/sys/sglist.h
==============================================================================
--- head/sys/sys/sglist.h	Sat May  2 20:47:58 2020	(r360568)
+++ head/sys/sys/sglist.h	Sat May  2 22:39:26 2020	(r360569)
@@ -57,7 +57,6 @@ struct sglist {
 
 struct bio;
 struct mbuf;
-struct mbuf_ext_pgs;
 struct uio;
 
 static __inline void
@@ -88,8 +87,8 @@ sglist_hold(struct sglist *sg)
 struct sglist *sglist_alloc(int nsegs, int mflags);
 int	sglist_append(struct sglist *sg, void *buf, size_t len);
 int	sglist_append_bio(struct sglist *sg, struct bio *bp);
-int	sglist_append_ext_pgs(struct sglist *sg, struct mbuf_ext_pgs *ext_pgs,
-	    size_t off, size_t len);
+int	sglist_append_ext_pgs(struct sglist *sg, struct mbuf *m, size_t off,
+	    size_t len);
 int	sglist_append_mb_ext_pgs(struct sglist *sg, struct mbuf *m);
 int	sglist_append_mbuf(struct sglist *sg, struct mbuf *m0);
 int	sglist_append_phys(struct sglist *sg, vm_paddr_t paddr,
@@ -105,8 +104,7 @@ struct sglist *sglist_build(void *buf, size_t len, int
 struct sglist *sglist_clone(struct sglist *sg, int mflags);
 int	sglist_consume_uio(struct sglist *sg, struct uio *uio, size_t resid);
 int	sglist_count(void *buf, size_t len);
-int	sglist_count_ext_pgs(struct mbuf_ext_pgs *ext_pgs, size_t off,
-	    size_t len);
+int	sglist_count_ext_pgs(struct mbuf *m, size_t off, size_t len);
 int	sglist_count_mb_ext_pgs(struct mbuf *m);
 int	sglist_count_vmpages(vm_page_t *m, size_t pgoff, size_t len);
 void	sglist_free(struct sglist *sg);



Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?202005022239.042MdR7Z036054>