Date: Wed, 11 Sep 2019 23:41:10 +0000 (UTC) From: Alexander Motin <mav@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-stable@freebsd.org, svn-src-stable-12@freebsd.org Subject: svn commit: r352234 - in stable/12/sys: crypto/aesni opencrypto Message-ID: <201909112341.x8BNfA8s030269@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: mav Date: Wed Sep 11 23:41:09 2019 New Revision: 352234 URL: https://svnweb.freebsd.org/changeset/base/352234 Log: MFC r342024 (by mmacy): Generalize AES iov optimization Right now, aesni_cipher_alloc does a bit of special-casing for CRYPTO_F_IOV, to not do any allocation if the first uio is large enough for the requested size. While working on ZFS crypto port, I ran into horrible performance because the code uses scatter-gather, and many of the times the data to encrypt was in the second entry. This code looks through the list, and tries to see if there is a single uio that can contain the requested data, and, if so, uses that. This has a slight impact on the current consumers, in that the check is a little more complicated for the ones that use CRYPTO_F_IOV -- but none of them meet the criteria for testing more than one. Modified: stable/12/sys/crypto/aesni/aesni.c stable/12/sys/opencrypto/criov.c stable/12/sys/opencrypto/cryptodev.h Directory Properties: stable/12/ (props changed) Modified: stable/12/sys/crypto/aesni/aesni.c ============================================================================== --- stable/12/sys/crypto/aesni/aesni.c Wed Sep 11 21:24:14 2019 (r352233) +++ stable/12/sys/crypto/aesni/aesni.c Wed Sep 11 23:41:09 2019 (r352234) @@ -403,29 +403,14 @@ static uint8_t * aesni_cipher_alloc(struct cryptodesc *enccrd, struct cryptop *crp, bool *allocated) { - struct mbuf *m; - struct uio *uio; - struct iovec *iov; uint8_t *addr; - if (crp->crp_flags & CRYPTO_F_IMBUF) { - m = (struct mbuf *)crp->crp_buf; - if (m->m_next != NULL) - goto alloc; - addr = mtod(m, uint8_t *); - } else if (crp->crp_flags & CRYPTO_F_IOV) { - uio = (struct uio *)crp->crp_buf; - if (uio->uio_iovcnt != 1) - goto alloc; - iov = uio->uio_iov; - addr = (uint8_t *)iov->iov_base; - } else - addr = (uint8_t *)crp->crp_buf; - *allocated = false; - addr += enccrd->crd_skip; - return (addr); - -alloc: + addr = crypto_contiguous_subsegment(crp->crp_flags, + crp->crp_buf, enccrd->crd_skip, enccrd->crd_len); + if (addr != NULL) { + *allocated = false; + return (addr); + } addr = malloc(enccrd->crd_len, M_AESNI, M_NOWAIT); if (addr != NULL) { *allocated = true; Modified: stable/12/sys/opencrypto/criov.c ============================================================================== --- stable/12/sys/opencrypto/criov.c Wed Sep 11 21:24:14 2019 (r352233) +++ stable/12/sys/opencrypto/criov.c Wed Sep 11 23:41:09 2019 (r352234) @@ -38,6 +38,8 @@ __FBSDID("$FreeBSD$"); #include <sys/kernel.h> #include <sys/mbuf.h> #include <sys/uio.h> +#include <sys/limits.h> +#include <sys/lock.h> #include <opencrypto/cryptodev.h> @@ -239,3 +241,55 @@ crypto_mbuftoiov(struct mbuf *mbuf, struct iovec **iov *cnt = i; return 0; } + +static inline void * +m_contiguous_subsegment(struct mbuf *m, size_t skip, size_t len) +{ + int rel_off; + + MPASS(skip <= INT_MAX); + + m = m_getptr(m, (int)skip, &rel_off); + if (m == NULL) + return (NULL); + + MPASS(rel_off >= 0); + skip = rel_off; + if (skip + len > m->m_len) + return (NULL); + + return (mtod(m, char*) + skip); +} + +static inline void * +cuio_contiguous_segment(struct uio *uio, size_t skip, size_t len) +{ + int rel_off, idx; + + MPASS(skip <= INT_MAX); + idx = cuio_getptr(uio, (int)skip, &rel_off); + if (idx < 0) + return (NULL); + + MPASS(rel_off >= 0); + skip = rel_off; + if (skip + len > uio->uio_iov[idx].iov_len) + return (NULL); + return ((char *)uio->uio_iov[idx].iov_base + skip); +} + +void * +crypto_contiguous_subsegment(int crp_flags, void *crpbuf, + size_t skip, size_t len) +{ + if ((crp_flags & CRYPTO_F_IMBUF) != 0) + return (m_contiguous_subsegment(crpbuf, skip, len)); + else if ((crp_flags & CRYPTO_F_IOV) != 0) + return (cuio_contiguous_segment(crpbuf, skip, len)); + else { + MPASS((crp_flags & (CRYPTO_F_IMBUF | CRYPTO_F_IOV)) != + (CRYPTO_F_IMBUF | CRYPTO_F_IOV)); + return ((char*)crpbuf + skip); + } +} + Modified: stable/12/sys/opencrypto/cryptodev.h ============================================================================== --- stable/12/sys/opencrypto/cryptodev.h Wed Sep 11 21:24:14 2019 (r352233) +++ stable/12/sys/opencrypto/cryptodev.h Wed Sep 11 23:41:09 2019 (r352234) @@ -571,5 +571,7 @@ extern void crypto_copydata(int flags, caddr_t buf, in extern int crypto_apply(int flags, caddr_t buf, int off, int len, int (*f)(void *, void *, u_int), void *arg); +extern void *crypto_contiguous_subsegment(int, void *, size_t, size_t); + #endif /* _KERNEL */ #endif /* _CRYPTO_CRYPTO_H_ */
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201909112341.x8BNfA8s030269>