Skip site navigation (1)Skip section navigation (2) 
Date:      Thu, 13 Dec 2018 04:40:53 +0000 (UTC)
From:      Matt Macy <mmacy@FreeBSD.org>
To:        src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org
Subject:   svn commit: r342024 - in head/sys: crypto/aesni opencrypto
Message-ID:  <201812130440.wBD4erR3014211@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help 
Author: mmacy
Date: Thu Dec 13 04:40:53 2018
New Revision: 342024
URL: https://svnweb.freebsd.org/changeset/base/342024

Log:
  Generalize AES iov optimization
  
  Right now, aesni_cipher_alloc does a bit of special-casing
  for CRYPTO_F_IOV, to not do any allocation if the first uio
  is large enough for the requested size. While working on ZFS
  crypto port, I ran into horrible performance because the code
  uses scatter-gather, and many of the times the data to encrypt
  was in the second entry. This code looks through the list, and
  tries to see if there is a single uio that can contain the
  requested data, and, if so, uses that.
  
  This has a slight impact on the current consumers, in that the
  check is a little more complicated for the ones that use
  CRYPTO_F_IOV -- but none of them meet the criteria for testing
  more than one.
  
  Submitted by:	sef at ixsystems.com
  Reviewed by:	cem@
  MFC after:	3 days
  Sponsored by:	iX Systems
  Differential Revision:	https://reviews.freebsd.org/D18522

Modified:
  head/sys/crypto/aesni/aesni.c
  head/sys/opencrypto/criov.c
  head/sys/opencrypto/cryptodev.h

Modified: head/sys/crypto/aesni/aesni.c
==============================================================================
--- head/sys/crypto/aesni/aesni.c	Thu Dec 13 03:12:15 2018	(r342023)
+++ head/sys/crypto/aesni/aesni.c	Thu Dec 13 04:40:53 2018	(r342024)
@@ -403,29 +403,14 @@ static uint8_t *
 aesni_cipher_alloc(struct cryptodesc *enccrd, struct cryptop *crp,
     bool *allocated)
 {
-	struct mbuf *m;
-	struct uio *uio;
-	struct iovec *iov;
 	uint8_t *addr;
 
-	if (crp->crp_flags & CRYPTO_F_IMBUF) {
-		m = (struct mbuf *)crp->crp_buf;
-		if (m->m_next != NULL)
-			goto alloc;
-		addr = mtod(m, uint8_t *);
-	} else if (crp->crp_flags & CRYPTO_F_IOV) {
-		uio = (struct uio *)crp->crp_buf;
-		if (uio->uio_iovcnt != 1)
-			goto alloc;
-		iov = uio->uio_iov;
-		addr = (uint8_t *)iov->iov_base;
-	} else
-		addr = (uint8_t *)crp->crp_buf;
-	*allocated = false;
-	addr += enccrd->crd_skip;
-	return (addr);
-
-alloc:
+	addr = crypto_contiguous_subsegment(crp->crp_flags,
+	    crp->crp_buf, enccrd->crd_skip, enccrd->crd_len);
+	if (addr != NULL) {
+		*allocated = false;
+		return (addr);
+	}
 	addr = malloc(enccrd->crd_len, M_AESNI, M_NOWAIT);
 	if (addr != NULL) {
 		*allocated = true;

Modified: head/sys/opencrypto/criov.c
==============================================================================
--- head/sys/opencrypto/criov.c	Thu Dec 13 03:12:15 2018	(r342023)
+++ head/sys/opencrypto/criov.c	Thu Dec 13 04:40:53 2018	(r342024)
@@ -38,6 +38,8 @@ __FBSDID("$FreeBSD$");
 #include <sys/kernel.h>
 #include <sys/mbuf.h>
 #include <sys/uio.h>
+#include <sys/limits.h>
+#include <sys/lock.h>
 
 #include <opencrypto/cryptodev.h>
 
@@ -239,3 +241,55 @@ crypto_mbuftoiov(struct mbuf *mbuf, struct iovec **iov
 	*cnt = i;
 	return 0;
 }
+
+static inline void *
+m_contiguous_subsegment(struct mbuf *m, size_t skip, size_t len)
+{
+	int rel_off;
+
+	MPASS(skip <= INT_MAX);
+
+	m = m_getptr(m, (int)skip, &rel_off);
+	if (m == NULL)
+		return (NULL);
+
+	MPASS(rel_off >= 0);
+	skip = rel_off;
+	if (skip + len > m->m_len)
+		return (NULL);
+
+	return (mtod(m, char*) + skip);
+}
+
+static inline void *
+cuio_contiguous_segment(struct uio *uio, size_t skip, size_t len)
+{
+	int rel_off, idx;
+
+	MPASS(skip <= INT_MAX);
+	idx = cuio_getptr(uio, (int)skip, &rel_off);
+	if (idx < 0)
+		return (NULL);
+
+	MPASS(rel_off >= 0);
+	skip = rel_off;
+	if (skip + len > uio->uio_iov[idx].iov_len)
+		return (NULL);
+	return ((char *)uio->uio_iov[idx].iov_base + skip);
+}
+
+void *
+crypto_contiguous_subsegment(int crp_flags, void *crpbuf,
+    size_t skip, size_t len)
+{
+	if ((crp_flags & CRYPTO_F_IMBUF) != 0)
+		return (m_contiguous_subsegment(crpbuf, skip, len));
+	else if ((crp_flags & CRYPTO_F_IOV) != 0)
+		return (cuio_contiguous_segment(crpbuf, skip, len));
+	else {
+		MPASS((crp_flags & (CRYPTO_F_IMBUF | CRYPTO_F_IOV)) !=
+		    (CRYPTO_F_IMBUF | CRYPTO_F_IOV));
+		return ((char*)crpbuf + skip);
+	}
+}
+

Modified: head/sys/opencrypto/cryptodev.h
==============================================================================
--- head/sys/opencrypto/cryptodev.h	Thu Dec 13 03:12:15 2018	(r342023)
+++ head/sys/opencrypto/cryptodev.h	Thu Dec 13 04:40:53 2018	(r342024)
@@ -564,5 +564,7 @@ extern	void crypto_copydata(int flags, caddr_t buf, in
 extern	int crypto_apply(int flags, caddr_t buf, int off, int len,
 	    int (*f)(void *, void *, u_int), void *arg);
 
+extern void *crypto_contiguous_subsegment(int, void *, size_t, size_t);
+
 #endif /* _KERNEL */
 #endif /* _CRYPTO_CRYPTO_H_ */

Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201812130440.wBD4erR3014211>