From nobody Thu Oct 21 17:07:11 2021 X-Original-To: dev-commits-src-branches@mlmmj.nyi.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2610:1c1:1:606c::19:1]) by mlmmj.nyi.freebsd.org (Postfix) with ESMTP id AF1C71806FF5; Thu, 21 Oct 2021 17:07:13 +0000 (UTC) (envelope-from git@FreeBSD.org) Received: from mxrelay.nyi.freebsd.org (mxrelay.nyi.freebsd.org [IPv6:2610:1c1:1:606c::19:3]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits) server-digest SHA256 client-signature RSA-PSS (4096 bits) client-digest SHA256) (Client CN "mxrelay.nyi.freebsd.org", Issuer "R3" (verified OK)) by mx1.freebsd.org (Postfix) with ESMTPS id 4HZv6S6q7Mz4h7B; Thu, 21 Oct 2021 17:07:12 +0000 (UTC) (envelope-from git@FreeBSD.org) Received: from gitrepo.freebsd.org (gitrepo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:5]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits) server-digest SHA256) (Client did not present a certificate) by mxrelay.nyi.freebsd.org (Postfix) with ESMTPS id 95F671EDBA; Thu, 21 Oct 2021 17:07:11 +0000 (UTC) (envelope-from git@FreeBSD.org) Received: from gitrepo.freebsd.org ([127.0.1.44]) by gitrepo.freebsd.org (8.16.1/8.16.1) with ESMTP id 19LH7BcJ080525; Thu, 21 Oct 2021 17:07:11 GMT (envelope-from git@gitrepo.freebsd.org) Received: (from git@localhost) by gitrepo.freebsd.org (8.16.1/8.16.1/Submit) id 19LH7Bs1080524; Thu, 21 Oct 2021 17:07:11 GMT (envelope-from git) Date: Thu, 21 Oct 2021 17:07:11 GMT Message-Id: <202110211707.19LH7Bs1080524@gitrepo.freebsd.org> To: src-committers@FreeBSD.org, dev-commits-src-all@FreeBSD.org, dev-commits-src-branches@FreeBSD.org From: John Baldwin Subject: git: 8460b90d28c0 - stable/13 - Support unmapped mbufs in crypto buffers. List-Id: Commits to the stable branches of the FreeBSD src repository List-Archive: https://lists.freebsd.org/archives/dev-commits-src-branches List-Help: List-Post: List-Subscribe: List-Unsubscribe: Sender: owner-dev-commits-src-branches@freebsd.org X-BeenThere: dev-commits-src-branches@freebsd.org MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit X-Git-Committer: jhb X-Git-Repository: src X-Git-Refname: refs/heads/stable/13 X-Git-Reftype: branch X-Git-Commit: 8460b90d28c027f48427eb640c038598e7ba4c56 Auto-Submitted: auto-generated X-ThisMailContainsUnwantedMimeParts: N The branch stable/13 has been updated by jhb: URL: https://cgit.FreeBSD.org/src/commit/?id=8460b90d28c027f48427eb640c038598e7ba4c56 commit 8460b90d28c027f48427eb640c038598e7ba4c56 Author: John Baldwin AuthorDate: 2021-05-25 23:59:18 +0000 Commit: John Baldwin CommitDate: 2021-10-21 15:51:26 +0000 Support unmapped mbufs in crypto buffers. Reviewed by: markj Sponsored by: Netflix Differential Revision: https://reviews.freebsd.org/D30134 (cherry picked from commit 1c8f4b3c9f9e8ca5823d153d3b117246b3d18db4) --- sys/opencrypto/criov.c | 121 +++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 111 insertions(+), 10 deletions(-) diff --git a/sys/opencrypto/criov.c b/sys/opencrypto/criov.c index b18a01faf26a..bf7965032084 100644 --- a/sys/opencrypto/criov.c +++ b/sys/opencrypto/criov.c @@ -239,6 +239,99 @@ cvm_page_copydata(vm_page_t *pages, int off, int len, caddr_t cp) } #endif /* CRYPTO_MAY_HAVE_VMPAGE */ +/* + * Given a starting page in an m_epg, determine the length of the + * current physically contiguous segment. + */ +static __inline size_t +m_epg_pages_extent(struct mbuf *m, int idx, u_int pglen) +{ + size_t len; + u_int i; + + len = pglen; + for (i = idx + 1; i < m->m_epg_npgs; i++) { + if (m->m_epg_pa[i - 1] + PAGE_SIZE != m->m_epg_pa[i]) + break; + len += m_epg_pagelen(m, i, 0); + } + return (len); +} + +static __inline void * +m_epg_segbase(struct mbuf *m, size_t offset) +{ + u_int i, pglen, pgoff; + + offset += mtod(m, vm_offset_t); + if (offset < m->m_epg_hdrlen) + return (m->m_epg_hdr + offset); + offset -= m->m_epg_hdrlen; + pgoff = m->m_epg_1st_off; + for (i = 0; i < m->m_epg_npgs; i++) { + pglen = m_epg_pagelen(m, i, pgoff); + if (offset < pglen) + return ((void *)PHYS_TO_DMAP(m->m_epg_pa[i] + pgoff + + offset)); + offset -= pglen; + pgoff = 0; + } + KASSERT(offset <= m->m_epg_trllen, ("%s: offset beyond trailer", + __func__)); + return (m->m_epg_trail + offset); +} + +static __inline size_t +m_epg_seglen(struct mbuf *m, size_t offset) +{ + u_int i, pglen, pgoff; + + offset += mtod(m, vm_offset_t); + if (offset < m->m_epg_hdrlen) + return (m->m_epg_hdrlen - offset); + offset -= m->m_epg_hdrlen; + pgoff = m->m_epg_1st_off; + for (i = 0; i < m->m_epg_npgs; i++) { + pglen = m_epg_pagelen(m, i, pgoff); + if (offset < pglen) + return (m_epg_pages_extent(m, i, pglen) - offset); + offset -= pglen; + pgoff = 0; + } + KASSERT(offset <= m->m_epg_trllen, ("%s: offset beyond trailer", + __func__)); + return (m->m_epg_trllen - offset); +} + +static __inline void * +m_epg_contiguous_subsegment(struct mbuf *m, size_t skip, size_t len) +{ + u_int i, pglen, pgoff; + + skip += mtod(m, vm_offset_t); + if (skip < m->m_epg_hdrlen) { + if (len > m->m_epg_hdrlen - skip) + return (NULL); + return (m->m_epg_hdr + skip); + } + skip -= m->m_epg_hdrlen; + pgoff = m->m_epg_1st_off; + for (i = 0; i < m->m_epg_npgs; i++) { + pglen = m_epg_pagelen(m, i, pgoff); + if (skip < pglen) { + if (len > m_epg_pages_extent(m, i, pglen) - skip) + return (NULL); + return ((void *)PHYS_TO_DMAP(m->m_epg_pa[i] + pgoff + + skip)); + } + skip -= pglen; + pgoff = 0; + } + KASSERT(skip <= m->m_epg_trllen && len <= m->m_epg_trllen - skip, + ("%s: segment beyond trailer", __func__)); + return (m->m_epg_trail + skip); +} + void crypto_cursor_init(struct crypto_buffer_cursor *cc, const struct crypto_buffer *cb) @@ -345,8 +438,8 @@ crypto_cursor_segbase(struct crypto_buffer_cursor *cc) case CRYPTO_BUF_MBUF: if (cc->cc_mbuf == NULL) return (NULL); - KASSERT((cc->cc_mbuf->m_flags & M_EXTPG) == 0, - ("%s: not supported for unmapped mbufs", __func__)); + if (cc->cc_mbuf->m_flags & M_EXTPG) + return (m_epg_segbase(cc->cc_mbuf, cc->cc_offset)); return (mtod(cc->cc_mbuf, char *) + cc->cc_offset); case CRYPTO_BUF_VMPAGE: return ((char *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS( @@ -372,6 +465,8 @@ crypto_cursor_seglen(struct crypto_buffer_cursor *cc) case CRYPTO_BUF_MBUF: if (cc->cc_mbuf == NULL) return (0); + if (cc->cc_mbuf->m_flags & M_EXTPG) + return (m_epg_seglen(cc->cc_mbuf, cc->cc_offset)); return (cc->cc_mbuf->m_len - cc->cc_offset); case CRYPTO_BUF_UIO: return (cc->cc_iov->iov_len - cc->cc_offset); @@ -401,12 +496,14 @@ crypto_cursor_copyback(struct crypto_buffer_cursor *cc, int size, break; case CRYPTO_BUF_MBUF: for (;;) { - KASSERT((cc->cc_mbuf->m_flags & M_EXTPG) == 0, - ("%s: not supported for unmapped mbufs", __func__)); - dst = mtod(cc->cc_mbuf, char *) + cc->cc_offset; + /* + * This uses m_copyback() for individual + * mbufs so that cc_mbuf and cc_offset are + * updated. + */ remain = cc->cc_mbuf->m_len - cc->cc_offset; todo = MIN(remain, size); - memcpy(dst, src, todo); + m_copyback(cc->cc_mbuf, cc->cc_offset, todo, src); src += todo; if (todo < remain) { cc->cc_offset += todo; @@ -482,12 +579,14 @@ crypto_cursor_copydata(struct crypto_buffer_cursor *cc, int size, void *vdst) break; case CRYPTO_BUF_MBUF: for (;;) { - KASSERT((cc->cc_mbuf->m_flags & M_EXTPG) == 0, - ("%s: not supported for unmapped mbufs", __func__)); - src = mtod(cc->cc_mbuf, const char *) + cc->cc_offset; + /* + * This uses m_copydata() for individual + * mbufs so that cc_mbuf and cc_offset are + * updated. + */ remain = cc->cc_mbuf->m_len - cc->cc_offset; todo = MIN(remain, size); - memcpy(dst, src, todo); + m_copydata(cc->cc_mbuf, cc->cc_offset, todo, dst); dst += todo; if (todo < remain) { cc->cc_offset += todo; @@ -715,6 +814,8 @@ m_contiguous_subsegment(struct mbuf *m, size_t skip, size_t len) if (skip + len > m->m_len) return (NULL); + if (m->m_flags & M_EXTPG) + return (m_epg_contiguous_subsegment(m, skip, len)); return (mtod(m, char*) + skip); }