Date: Mon, 27 Apr 2020 23:59:42 +0000 (UTC) From: John Baldwin <jhb@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r360417 - head/sys/dev/cxgbe/tom Message-ID: <202004272359.03RNxggs083175@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: jhb Date: Mon Apr 27 23:59:42 2020 New Revision: 360417 URL: https://svnweb.freebsd.org/changeset/base/360417 Log: Add support for KTLS RX over TOE to T6. This largely reuses the TLS TOE support added in r330884. However, this uses the KTLS framework in upstream OpenSSL rather than requiring Chelsio-specific patches to OpenSSL. As with the existing TLS TOE support, use of RX offload requires setting the tls_rx_ports sysctl. Reviewed by: np Sponsored by: Chelsio Communications Differential Revision: https://reviews.freebsd.org/D24453 Modified: head/sys/dev/cxgbe/tom/t4_tls.c head/sys/dev/cxgbe/tom/t4_tom.c head/sys/dev/cxgbe/tom/t4_tom.h Modified: head/sys/dev/cxgbe/tom/t4_tls.c ============================================================================== --- head/sys/dev/cxgbe/tom/t4_tls.c Mon Apr 27 23:55:09 2020 (r360416) +++ head/sys/dev/cxgbe/tom/t4_tls.c Mon Apr 27 23:59:42 2020 (r360417) @@ -379,7 +379,7 @@ prepare_rxkey_wr(struct tls_keyctx *kwr, struct tls_ke int proto_ver = kctx->proto_ver; kwr->u.rxhdr.flitcnt_hmacctrl = - ((kctx->tx_key_info_size >> 4) << 3) | kctx->hmac_ctrl; + ((kctx->rx_key_info_size >> 4) << 3) | kctx->hmac_ctrl; kwr->u.rxhdr.protover_ciphmode = V_TLS_KEYCTX_TX_WR_PROTOVER(get_proto_ver(proto_ver)) | @@ -408,7 +408,7 @@ prepare_rxkey_wr(struct tls_keyctx *kwr, struct tls_ke (IPAD_SIZE + OPAD_SIZE)); } else { memcpy(kwr->keys.edkey, kctx->rx.key, - (kctx->tx_key_info_size - SALT_SIZE)); + (kctx->rx_key_info_size - SALT_SIZE)); memcpy(kwr->u.rxhdr.rxsalt, kctx->rx.salt, SALT_SIZE); } } @@ -674,6 +674,13 @@ program_key_context(struct tcpcb *tp, struct toepcb *t if ((G_KEY_GET_LOC(k_ctx->l_p_key) == KEY_WRITE_RX) || (tls_ofld->key_location == TLS_SFO_WR_CONTEXTLOC_DDR)) { + + /* + * XXX: The userland library sets tx_key_info_size, not + * rx_key_info_size. + */ + k_ctx->rx_key_info_size = k_ctx->tx_key_info_size; + error = tls_program_key_id(toep, k_ctx); if (error) { /* XXX: Only clear quiesce for KEY_WRITE_RX? */ @@ -866,31 +873,37 @@ t4_ctloutput_tls(struct socket *so, struct sockopt *so #ifdef KERN_TLS static void -init_ktls_key_context(struct ktls_session *tls, struct tls_key_context *k_ctx) +init_ktls_key_context(struct ktls_session *tls, struct tls_key_context *k_ctx, + int direction) { struct auth_hash *axf; - u_int mac_key_size; - char *hash; + u_int key_info_size, mac_key_size; + char *hash, *key; - k_ctx->l_p_key = V_KEY_GET_LOC(KEY_WRITE_TX); - if (tls->params.tls_vminor == TLS_MINOR_VER_ONE) - k_ctx->proto_ver = SCMD_PROTO_VERSION_TLS_1_1; - else - k_ctx->proto_ver = SCMD_PROTO_VERSION_TLS_1_2; + k_ctx->l_p_key = V_KEY_GET_LOC(direction == KTLS_TX ? KEY_WRITE_TX : + KEY_WRITE_RX); + k_ctx->proto_ver = tls->params.tls_vmajor << 8 | tls->params.tls_vminor; k_ctx->cipher_secret_size = tls->params.cipher_key_len; - k_ctx->tx_key_info_size = sizeof(struct tx_keyctx_hdr) + + key_info_size = sizeof(struct tx_keyctx_hdr) + k_ctx->cipher_secret_size; - memcpy(k_ctx->tx.key, tls->params.cipher_key, - tls->params.cipher_key_len); - hash = k_ctx->tx.key + tls->params.cipher_key_len; + if (direction == KTLS_TX) + key = k_ctx->tx.key; + else + key = k_ctx->rx.key; + memcpy(key, tls->params.cipher_key, tls->params.cipher_key_len); + hash = key + tls->params.cipher_key_len; if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) { k_ctx->state.auth_mode = SCMD_AUTH_MODE_GHASH; k_ctx->state.enc_mode = SCMD_CIPH_MODE_AES_GCM; k_ctx->iv_size = 4; k_ctx->mac_first = 0; k_ctx->hmac_ctrl = SCMD_HMAC_CTRL_NOP; - k_ctx->tx_key_info_size += GMAC_BLOCK_LEN; - memcpy(k_ctx->tx.salt, tls->params.iv, SALT_SIZE); + key_info_size += GMAC_BLOCK_LEN; + k_ctx->mac_secret_size = 0; + if (direction == KTLS_TX) + memcpy(k_ctx->tx.salt, tls->params.iv, SALT_SIZE); + else + memcpy(k_ctx->rx.salt, tls->params.iv, SALT_SIZE); t4_init_gmac_hash(tls->params.cipher_key, tls->params.cipher_key_len, hash); } else { @@ -917,29 +930,38 @@ init_ktls_key_context(struct ktls_session *tls, struct k_ctx->iv_size = 8; /* for CBC, iv is 16B, unit of 2B */ k_ctx->mac_first = 1; k_ctx->hmac_ctrl = SCMD_HMAC_CTRL_NO_TRUNC; - k_ctx->tx_key_info_size += roundup2(mac_key_size, 16) * 2; + key_info_size += roundup2(mac_key_size, 16) * 2; k_ctx->mac_secret_size = mac_key_size; t4_init_hmac_digest(axf, mac_key_size, tls->params.auth_key, tls->params.auth_key_len, hash); } + if (direction == KTLS_TX) + k_ctx->tx_key_info_size = key_info_size; + else + k_ctx->rx_key_info_size = key_info_size; k_ctx->frag_size = tls->params.max_frame_len; k_ctx->iv_ctrl = 1; } int -tls_alloc_ktls(struct toepcb *toep, struct ktls_session *tls) +tls_alloc_ktls(struct toepcb *toep, struct ktls_session *tls, int direction) { + struct adapter *sc = td_adapter(toep->td); struct tls_key_context *k_ctx; - int error; + int error, key_offset; if (toep->tls.mode == TLS_MODE_TLSOM) return (EINVAL); if (!can_tls_offload(td_adapter(toep->td))) return (EINVAL); switch (ulp_mode(toep)) { + case ULP_MODE_TLS: + break; case ULP_MODE_NONE: case ULP_MODE_TCPDDP: + if (direction != KTLS_TX) + return (EINVAL); break; default: return (EINVAL); @@ -987,48 +1009,81 @@ tls_alloc_ktls(struct toepcb *toep, struct ktls_sessio tls->params.tls_vminor > TLS_MINOR_VER_TWO) return (EPROTONOSUPPORT); + /* Bail if we already have a key. */ + if (direction == KTLS_TX) { + if (toep->tls.tx_key_addr != -1) + return (EOPNOTSUPP); + } else { + if (toep->tls.rx_key_addr != -1) + return (EOPNOTSUPP); + } + /* * XXX: This assumes no key renegotation. If KTLS ever supports * that we will want to allocate TLS sessions dynamically rather * than as a static member of toep. */ k_ctx = &toep->tls.k_ctx; - init_ktls_key_context(tls, k_ctx); + init_ktls_key_context(tls, k_ctx, direction); - toep->tls.scmd0.seqno_numivs = - (V_SCMD_SEQ_NO_CTRL(3) | - V_SCMD_PROTO_VERSION(k_ctx->proto_ver) | - V_SCMD_ENC_DEC_CTRL(SCMD_ENCDECCTRL_ENCRYPT) | - V_SCMD_CIPH_AUTH_SEQ_CTRL((k_ctx->mac_first == 0)) | - V_SCMD_CIPH_MODE(k_ctx->state.enc_mode) | - V_SCMD_AUTH_MODE(k_ctx->state.auth_mode) | - V_SCMD_HMAC_CTRL(k_ctx->hmac_ctrl) | - V_SCMD_IV_SIZE(k_ctx->iv_size)); + error = tls_program_key_id(toep, k_ctx); + if (error) + return (error); - toep->tls.scmd0.ivgen_hdrlen = - (V_SCMD_IV_GEN_CTRL(k_ctx->iv_ctrl) | - V_SCMD_KEY_CTX_INLINE(0) | - V_SCMD_TLS_FRAG_ENABLE(1)); + if (direction == KTLS_TX) { + toep->tls.scmd0.seqno_numivs = + (V_SCMD_SEQ_NO_CTRL(3) | + V_SCMD_PROTO_VERSION(get_proto_ver(k_ctx->proto_ver)) | + V_SCMD_ENC_DEC_CTRL(SCMD_ENCDECCTRL_ENCRYPT) | + V_SCMD_CIPH_AUTH_SEQ_CTRL((k_ctx->mac_first == 0)) | + V_SCMD_CIPH_MODE(k_ctx->state.enc_mode) | + V_SCMD_AUTH_MODE(k_ctx->state.auth_mode) | + V_SCMD_HMAC_CTRL(k_ctx->hmac_ctrl) | + V_SCMD_IV_SIZE(k_ctx->iv_size)); - if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) - toep->tls.iv_len = 8; - else - toep->tls.iv_len = AES_BLOCK_LEN; + toep->tls.scmd0.ivgen_hdrlen = + (V_SCMD_IV_GEN_CTRL(k_ctx->iv_ctrl) | + V_SCMD_KEY_CTX_INLINE(0) | + V_SCMD_TLS_FRAG_ENABLE(1)); - toep->tls.mac_length = k_ctx->mac_secret_size; + if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) + toep->tls.iv_len = 8; + else + toep->tls.iv_len = AES_BLOCK_LEN; - toep->tls.tx_key_addr = -1; + toep->tls.mac_length = k_ctx->mac_secret_size; - error = tls_program_key_id(toep, k_ctx); - if (error) - return (error); + toep->tls.fcplenmax = get_tp_plen_max(&toep->tls); + toep->tls.expn_per_ulp = tls->params.tls_hlen + + tls->params.tls_tlen; + toep->tls.pdus_per_ulp = 1; + toep->tls.adjusted_plen = toep->tls.expn_per_ulp + + toep->tls.k_ctx.frag_size; + } else { + /* Stop timer on handshake completion */ + tls_stop_handshake_timer(toep); - toep->tls.fcplenmax = get_tp_plen_max(&toep->tls); - toep->tls.expn_per_ulp = tls->params.tls_hlen + tls->params.tls_tlen; - toep->tls.pdus_per_ulp = 1; - toep->tls.adjusted_plen = toep->tls.expn_per_ulp + - toep->tls.k_ctx.frag_size; + toep->flags &= ~TPF_FORCE_CREDITS; + /* + * RX key tags are an index into the key portion of MA + * memory stored as an offset from the base address in + * units of 64 bytes. + */ + key_offset = toep->tls.rx_key_addr - sc->vres.key.start; + t4_set_tls_keyid(toep, key_offset / 64); + t4_set_tls_tcb_field(toep, W_TCB_ULP_RAW, + V_TCB_ULP_RAW(M_TCB_ULP_RAW), + V_TCB_ULP_RAW((V_TF_TLS_KEY_SIZE(3) | + V_TF_TLS_CONTROL(1) | + V_TF_TLS_ACTIVE(1) | + V_TF_TLS_ENABLE(1)))); + t4_set_tls_tcb_field(toep, W_TCB_TLS_SEQ, + V_TCB_TLS_SEQ(M_TCB_TLS_SEQ), + V_TCB_TLS_SEQ(0)); + t4_clear_rx_quiesce(toep); + } + toep->tls.mode = TLS_MODE_KTLS; return (0); @@ -1671,7 +1726,7 @@ t4_push_ktls(struct adapter *sc, struct toepcb *toep, ("%s: flowc_wr not sent for tid %u.", __func__, toep->tid)); KASSERT(ulp_mode(toep) == ULP_MODE_NONE || - ulp_mode(toep) == ULP_MODE_TCPDDP, + ulp_mode(toep) == ULP_MODE_TCPDDP || ulp_mode(toep) == ULP_MODE_TLS, ("%s: ulp_mode %u for toep %p", __func__, ulp_mode(toep), toep)); KASSERT(tls_tx_key(toep), ("%s: TX key not set for toep %p", __func__, toep)); @@ -1956,6 +2011,10 @@ do_rx_tls_cmp(struct sge_iq *iq, const struct rss_head struct socket *so; struct sockbuf *sb; struct mbuf *tls_data; +#ifdef KERN_TLS + struct tls_get_record *tgr; + struct mbuf *control; +#endif int len, pdu_length, rx_credits; KASSERT(toep->tid == tid, ("%s: toep tid/atid mismatch", __func__)); @@ -1982,6 +2041,7 @@ do_rx_tls_cmp(struct sge_iq *iq, const struct rss_head pdu_length = G_CPL_RX_TLS_CMP_PDULENGTH(be32toh(cpl->pdulength_length)); + so = inp_inpcbtosocket(inp); tp = intotcpcb(inp); #ifdef VERBOSE_TRACES @@ -2006,35 +2066,94 @@ do_rx_tls_cmp(struct sge_iq *iq, const struct rss_head ("%s: payload too small", __func__)); tls_hdr_pkt = mtod(m, void *); - /* - * Only the TLS header is sent to OpenSSL, so report errors by - * altering the record type. - */ - if ((tls_hdr_pkt->res_to_mac_error & M_TLSRX_HDR_PKT_ERROR) != 0) - tls_hdr_pkt->type = CONTENT_TYPE_ERROR; - - /* Trim this CPL's mbuf to only include the TLS header. */ - KASSERT(m->m_len == len && m->m_next == NULL, - ("%s: CPL spans multiple mbufs", __func__)); - m->m_len = TLS_HEADER_LENGTH; - m->m_pkthdr.len = TLS_HEADER_LENGTH; - tls_data = mbufq_dequeue(&toep->ulp_pdu_reclaimq); if (tls_data != NULL) { KASSERT(be32toh(cpl->seq) == tls_data->m_pkthdr.tls_tcp_seq, ("%s: sequence mismatch", __func__)); + } +#ifdef KERN_TLS + if (toep->tls.mode == TLS_MODE_KTLS) { + /* Report decryption errors as EBADMSG. */ + if ((tls_hdr_pkt->res_to_mac_error & M_TLSRX_HDR_PKT_ERROR) != + 0) { + m_freem(m); + m_freem(tls_data); + + CURVNET_SET(toep->vnet); + so->so_error = EBADMSG; + sorwakeup(so); + + INP_WUNLOCK(inp); + CURVNET_RESTORE(); + + return (0); + } + + /* Allocate the control message mbuf. */ + control = sbcreatecontrol(NULL, sizeof(*tgr), TLS_GET_RECORD, + IPPROTO_TCP); + if (control == NULL) { + m_freem(m); + m_freem(tls_data); + + CURVNET_SET(toep->vnet); + so->so_error = ENOBUFS; + sorwakeup(so); + + INP_WUNLOCK(inp); + CURVNET_RESTORE(); + + return (0); + } + + tgr = (struct tls_get_record *) + CMSG_DATA(mtod(control, struct cmsghdr *)); + tgr->tls_type = tls_hdr_pkt->type; + tgr->tls_vmajor = be16toh(tls_hdr_pkt->version) >> 8; + tgr->tls_vminor = be16toh(tls_hdr_pkt->version) & 0xff; + + m_freem(m); + + if (tls_data != NULL) { + m_last(tls_data)->m_flags |= M_EOR; + tgr->tls_length = htobe16(tls_data->m_pkthdr.len); + } else + tgr->tls_length = 0; + m = tls_data; + } else +#endif + { /* - * Update the TLS header length to be the length of - * the payload data. + * Only the TLS header is sent to OpenSSL, so report + * errors by altering the record type. */ - tls_hdr_pkt->length = htobe16(tls_data->m_pkthdr.len); + if ((tls_hdr_pkt->res_to_mac_error & M_TLSRX_HDR_PKT_ERROR) != + 0) + tls_hdr_pkt->type = CONTENT_TYPE_ERROR; - m->m_next = tls_data; - m->m_pkthdr.len += tls_data->m_len; + /* Trim this CPL's mbuf to only include the TLS header. */ + KASSERT(m->m_len == len && m->m_next == NULL, + ("%s: CPL spans multiple mbufs", __func__)); + m->m_len = TLS_HEADER_LENGTH; + m->m_pkthdr.len = TLS_HEADER_LENGTH; + + if (tls_data != NULL) { + /* + * Update the TLS header length to be the length of + * the payload data. + */ + tls_hdr_pkt->length = htobe16(tls_data->m_pkthdr.len); + + m->m_next = tls_data; + m->m_pkthdr.len += tls_data->m_len; + } + +#ifdef KERN_TLS + control = NULL; +#endif } - so = inp_inpcbtosocket(inp); sb = &so->so_rcv; SOCKBUF_LOCK(sb); @@ -2044,6 +2163,9 @@ do_rx_tls_cmp(struct sge_iq *iq, const struct rss_head CTR3(KTR_CXGBE, "%s: tid %u, excess rx (%d bytes)", __func__, tid, pdu_length); m_freem(m); +#ifdef KERN_TLS + m_freem(control); +#endif SOCKBUF_UNLOCK(sb); INP_WUNLOCK(inp); @@ -2080,7 +2202,12 @@ do_rx_tls_cmp(struct sge_iq *iq, const struct rss_head sb->sb_flags &= ~SB_AUTOSIZE; } - sbappendstream_locked(sb, m, 0); +#ifdef KERN_TLS + if (control != NULL) + sbappendcontrol_locked(sb, m, control, 0); + else +#endif + sbappendstream_locked(sb, m, 0); rx_credits = sbspace(sb) > tp->rcv_wnd ? sbspace(sb) - tp->rcv_wnd : 0; #ifdef VERBOSE_TRACES CTR4(KTR_CXGBE, "%s: tid %u rx_credits %u rcv_wnd %u", Modified: head/sys/dev/cxgbe/tom/t4_tom.c ============================================================================== --- head/sys/dev/cxgbe/tom/t4_tom.c Mon Apr 27 23:55:09 2020 (r360416) +++ head/sys/dev/cxgbe/tom/t4_tom.c Mon Apr 27 23:59:42 2020 (r360417) @@ -40,9 +40,6 @@ __FBSDID("$FreeBSD$"); #include <sys/systm.h> #include <sys/kernel.h> #include <sys/ktr.h> -#ifdef KERN_TLS -#include <sys/ktls.h> -#endif #include <sys/lock.h> #include <sys/limits.h> #include <sys/module.h> @@ -824,10 +821,7 @@ t4_alloc_tls_session(struct toedev *tod, struct tcpcb INP_WLOCK_ASSERT(tp->t_inpcb); MPASS(tls != NULL); - if (direction != KTLS_TX) - return (EOPNOTSUPP); - - return (tls_alloc_ktls(toep, tls)); + return (tls_alloc_ktls(toep, tls, direction)); } #endif Modified: head/sys/dev/cxgbe/tom/t4_tom.h ============================================================================== --- head/sys/dev/cxgbe/tom/t4_tom.h Mon Apr 27 23:55:09 2020 (r360416) +++ head/sys/dev/cxgbe/tom/t4_tom.h Mon Apr 27 23:59:42 2020 (r360417) @@ -452,6 +452,6 @@ int tls_rx_key(struct toepcb *); void tls_stop_handshake_timer(struct toepcb *); int tls_tx_key(struct toepcb *); void tls_uninit_toep(struct toepcb *); -int tls_alloc_ktls(struct toepcb *, struct ktls_session *); +int tls_alloc_ktls(struct toepcb *, struct ktls_session *, int); #endif
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?202004272359.03RNxggs083175>