Skip site navigation (1)Skip section navigation (2) 
Date:      Sat, 2 May 2020 23:38:13 +0000 (UTC)
From:      Gleb Smirnoff <glebius@FreeBSD.org>
To:        src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org
Subject:   svn commit: r360573 - in head/sys: kern sys
Message-ID:  <202005022338.042NcD0F072681@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help 
Author: glebius
Date: Sat May  2 23:38:13 2020
New Revision: 360573
URL: https://svnweb.freebsd.org/changeset/base/360573

Log:
  Step 2.1: Build TLS workqueue from mbufs, not struct mbuf_ext_pgs.
  
  Reviewed by:	gallatin
  Differential Revision:	https://reviews.freebsd.org/D24598

Modified:
  head/sys/kern/kern_mbuf.c
  head/sys/kern/uipc_ktls.c
  head/sys/sys/ktls.h
  head/sys/sys/mbuf.h

Modified: head/sys/kern/kern_mbuf.c
==============================================================================
--- head/sys/kern/kern_mbuf.c	Sat May  2 22:56:22 2020	(r360572)
+++ head/sys/kern/kern_mbuf.c	Sat May  2 23:38:13 2020	(r360573)
@@ -1246,7 +1246,6 @@ mb_free_ext(struct mbuf *m)
 			break;
 		case EXT_PGS: {
 #ifdef KERN_TLS
-			struct mbuf_ext_pgs *pgs;
 			struct ktls_session *tls;
 #endif
 
@@ -1254,11 +1253,10 @@ mb_free_ext(struct mbuf *m)
 			    ("%s: ext_free not set", __func__));
 			mref->m_ext.ext_free(mref);
 #ifdef KERN_TLS
-			pgs = &mref->m_ext_pgs;
-			tls = pgs->tls;
+			tls = mref->m_ext_pgs.tls;
 			if (tls != NULL &&
 			    !refcount_release_if_not_last(&tls->refcount))
-				ktls_enqueue_to_free(pgs);
+				ktls_enqueue_to_free(mref);
 			else
 #endif
 				uma_zfree(zone_mbuf, mref);

Modified: head/sys/kern/uipc_ktls.c
==============================================================================
--- head/sys/kern/uipc_ktls.c	Sat May  2 22:56:22 2020	(r360572)
+++ head/sys/kern/uipc_ktls.c	Sat May  2 23:38:13 2020	(r360573)
@@ -79,7 +79,7 @@ __FBSDID("$FreeBSD$");
 
 struct ktls_wq {
 	struct mtx	mtx;
-	STAILQ_HEAD(, mbuf_ext_pgs) head;
+	STAILQ_HEAD(, mbuf) head;
 	bool		running;
 } __aligned(CACHE_LINE_SIZE);
 
@@ -1430,16 +1430,19 @@ ktls_frame(struct mbuf *top, struct ktls_session *tls,
 }
 
 void
-ktls_enqueue_to_free(struct mbuf_ext_pgs *pgs)
+ktls_enqueue_to_free(struct mbuf *m)
 {
+	struct mbuf_ext_pgs *pgs;
 	struct ktls_wq *wq;
 	bool running;
 
+	pgs = &m->m_ext_pgs;
+
 	/* Mark it for freeing. */
 	pgs->flags |= EPG_FLAG_2FREE;
 	wq = &ktls_wq[pgs->tls->wq_index];
 	mtx_lock(&wq->mtx);
-	STAILQ_INSERT_TAIL(&wq->head, pgs, stailq);
+	STAILQ_INSERT_TAIL(&wq->head, m, m_ext_pgs.stailq);
 	running = wq->running;
 	mtx_unlock(&wq->mtx);
 	if (!running)
@@ -1472,7 +1475,7 @@ ktls_enqueue(struct mbuf *m, struct socket *so, int pa
 
 	wq = &ktls_wq[pgs->tls->wq_index];
 	mtx_lock(&wq->mtx);
-	STAILQ_INSERT_TAIL(&wq->head, pgs, stailq);
+	STAILQ_INSERT_TAIL(&wq->head, m, m_ext_pgs.stailq);
 	running = wq->running;
 	mtx_unlock(&wq->mtx);
 	if (!running)
@@ -1481,11 +1484,12 @@ ktls_enqueue(struct mbuf *m, struct socket *so, int pa
 }
 
 static __noinline void
-ktls_encrypt(struct mbuf_ext_pgs *pgs)
+ktls_encrypt(struct mbuf *top)
 {
 	struct ktls_session *tls;
 	struct socket *so;
-	struct mbuf *m, *top;
+	struct mbuf *m;
+	struct mbuf_ext_pgs *pgs;
 	vm_paddr_t parray[1 + btoc(TLS_MAX_MSG_SIZE_V10_2)];
 	struct iovec src_iov[1 + btoc(TLS_MAX_MSG_SIZE_V10_2)];
 	struct iovec dst_iov[1 + btoc(TLS_MAX_MSG_SIZE_V10_2)];
@@ -1493,15 +1497,14 @@ ktls_encrypt(struct mbuf_ext_pgs *pgs)
 	int error, i, len, npages, off, total_pages;
 	bool is_anon;
 
-	so = pgs->so;
-	tls = pgs->tls;
-	top = __containerof(pgs, struct mbuf, m_ext_pgs);
-	KASSERT(tls != NULL, ("tls = NULL, top = %p, pgs = %p\n", top, pgs));
-	KASSERT(so != NULL, ("so = NULL, top = %p, pgs = %p\n", top, pgs));
+	so = top->m_ext_pgs.so;
+	tls = top->m_ext_pgs.tls;
+	KASSERT(tls != NULL, ("tls = NULL, top = %p\n", top));
+	KASSERT(so != NULL, ("so = NULL, top = %p\n", top));
 #ifdef INVARIANTS
-	pgs->so = NULL;
+	top->m_ext_pgs.so = NULL;
 #endif
-	total_pages = pgs->enc_cnt;
+	total_pages = top->m_ext_pgs.enc_cnt;
 	npages = 0;
 
 	/*
@@ -1631,10 +1634,8 @@ static void
 ktls_work_thread(void *ctx)
 {
 	struct ktls_wq *wq = ctx;
-	struct mbuf_ext_pgs *p, *n;
-	struct ktls_session *tls;
-	struct mbuf *m;
-	STAILQ_HEAD(, mbuf_ext_pgs) local_head;
+	struct mbuf *m, *n;
+	STAILQ_HEAD(, mbuf) local_head;
 
 #if defined(__aarch64__) || defined(__amd64__) || defined(__i386__)
 	fpu_kern_thread(0);
@@ -1651,14 +1652,12 @@ ktls_work_thread(void *ctx)
 		STAILQ_CONCAT(&local_head, &wq->head);
 		mtx_unlock(&wq->mtx);
 
-		STAILQ_FOREACH_SAFE(p, &local_head, stailq, n) {
-			if (p->flags & EPG_FLAG_2FREE) {
-				tls = p->tls;
-				ktls_free(tls);
-				m = __containerof(p, struct mbuf, m_ext_pgs);
+		STAILQ_FOREACH_SAFE(m, &local_head, m_ext_pgs.stailq, n) {
+			if (m->m_ext_pgs.flags & EPG_FLAG_2FREE) {
+				ktls_free(m->m_ext_pgs.tls);
 				uma_zfree(zone_mbuf, m);
 			} else {
-				ktls_encrypt(p);
+				ktls_encrypt(m);
 				counter_u64_add(ktls_cnt_on, -1);
 			}
 		}

Modified: head/sys/sys/ktls.h
==============================================================================
--- head/sys/sys/ktls.h	Sat May  2 22:56:22 2020	(r360572)
+++ head/sys/sys/ktls.h	Sat May  2 23:38:13 2020	(r360573)
@@ -169,7 +169,6 @@ struct iovec;
 struct ktls_session;
 struct m_snd_tag;
 struct mbuf;
-struct mbuf_ext_pgs;
 struct sockbuf;
 struct socket;
 
@@ -212,7 +211,7 @@ void ktls_frame(struct mbuf *m, struct ktls_session *t
     uint8_t record_type);
 void ktls_seq(struct sockbuf *sb, struct mbuf *m);
 void ktls_enqueue(struct mbuf *m, struct socket *so, int page_count);
-void ktls_enqueue_to_free(struct mbuf_ext_pgs *pgs);
+void ktls_enqueue_to_free(struct mbuf *m);
 int ktls_get_rx_mode(struct socket *so);
 int ktls_set_tx_mode(struct socket *so, int mode);
 int ktls_get_tx_mode(struct socket *so);

Modified: head/sys/sys/mbuf.h
==============================================================================
--- head/sys/sys/mbuf.h	Sat May  2 22:56:22 2020	(r360572)
+++ head/sys/sys/mbuf.h	Sat May  2 23:38:13 2020	(r360573)
@@ -372,7 +372,7 @@ struct mbuf {
 					struct ktls_session *tls;
 					struct socket	*so;
 					uint64_t	seqno;
-					STAILQ_ENTRY(mbuf_ext_pgs) stailq;
+					STAILQ_ENTRY(mbuf) stailq;
 				} m_ext_pgs;
 			};
 			union {

Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?202005022338.042NcD0F072681>