Date: Fri, 29 Apr 2022 23:12:09 GMT From: John Baldwin <jhb@FreeBSD.org> To: src-committers@FreeBSD.org, dev-commits-src-all@FreeBSD.org, dev-commits-src-branches@FreeBSD.org Subject: git: 1462dc95f796 - stable/13 - ktls: Support for TLS 1.3 receive offload. Message-ID: <202204292312.23TNC9H3044263@gitrepo.freebsd.org>
next in thread | raw e-mail | index | archive | help
The branch stable/13 has been updated by jhb: URL: https://cgit.FreeBSD.org/src/commit/?id=1462dc95f796295f442975f2ca70f136774551ef commit 1462dc95f796295f442975f2ca70f136774551ef Author: John Baldwin <jhb@FreeBSD.org> AuthorDate: 2021-12-14 19:01:05 +0000 Commit: John Baldwin <jhb@FreeBSD.org> CommitDate: 2022-04-29 21:07:27 +0000 ktls: Support for TLS 1.3 receive offload. Note that support for TLS 1.3 receive offload in OpenSSL is still an open pull request in active development. However, potential changes to that pull request should not affect the kernel interface. Reviewed by: hselasky Sponsored by: Netflix Differential Revision: https://reviews.freebsd.org/D33007 (cherry picked from commit 05a1d0f5d7ac8400975d1eaa30a718a1ff48b139) --- sys/kern/uipc_ktls.c | 72 ++++++++++++++++++++++--- sys/opencrypto/ktls_ocf.c | 77 +++++++++++++++++++++++---- tests/sys/kern/ktls_test.c | 129 +++++++++++++++++++++++++++++++++------------ 3 files changed, 226 insertions(+), 52 deletions(-) diff --git a/sys/kern/uipc_ktls.c b/sys/kern/uipc_ktls.c index 50bd0a0df21e..6d436c72de2f 100644 --- a/sys/kern/uipc_ktls.c +++ b/sys/kern/uipc_ktls.c @@ -1088,11 +1088,6 @@ ktls_enable_rx(struct socket *so, struct tls_enable *en) if (en->cipher_algorithm == CRYPTO_AES_CBC && !ktls_cbc_enable) return (ENOTSUP); - /* TLS 1.3 is not yet supported. */ - if (en->tls_vmajor == TLS_MAJOR_VER_ONE && - en->tls_vminor == TLS_MINOR_VER_THREE) - return (ENOTSUP); - error = ktls_create_session(so, en, &tls); if (error) return (error); @@ -1822,6 +1817,53 @@ out: return (top); } +/* + * Determine the length of the trailing zero padding and find the real + * record type in the byte before the padding. + * + * Walking the mbuf chain backwards is clumsy, so another option would + * be to scan forwards remembering the last non-zero byte before the + * trailer. However, it would be expensive to scan the entire record. + * Instead, find the last non-zero byte of each mbuf in the chain + * keeping track of the relative offset of that nonzero byte. + * + * trail_len is the size of the MAC/tag on input and is set to the + * size of the full trailer including padding and the record type on + * return. + */ +static int +tls13_find_record_type(struct ktls_session *tls, struct mbuf *m, int tls_len, + int *trailer_len, uint8_t *record_typep) +{ + char *cp; + u_int digest_start, last_offset, m_len, offset; + uint8_t record_type; + + digest_start = tls_len - *trailer_len; + last_offset = 0; + offset = 0; + for (; m != NULL && offset < digest_start; + offset += m->m_len, m = m->m_next) { + /* Don't look for padding in the tag. */ + m_len = min(digest_start - offset, m->m_len); + cp = mtod(m, char *); + + /* Find last non-zero byte in this mbuf. */ + while (m_len > 0 && cp[m_len - 1] == 0) + m_len--; + if (m_len > 0) { + record_type = cp[m_len - 1]; + last_offset = offset + m_len; + } + } + if (last_offset < tls->params.tls_hlen) + return (EBADMSG); + + *record_typep = record_type; + *trailer_len = tls_len - last_offset + 1; + return (0); +} + static void ktls_decrypt(struct socket *so) { @@ -1833,6 +1875,8 @@ ktls_decrypt(struct socket *so) struct mbuf *control, *data, *m; uint64_t seqno; int error, remain, tls_len, trail_len; + bool tls13; + uint8_t vminor, record_type; hdr = (struct tls_record_layer *)tls_header; sb = &so->so_rcv; @@ -1843,6 +1887,11 @@ ktls_decrypt(struct socket *so) tls = sb->sb_tls_info; MPASS(tls != NULL); + tls13 = (tls->params.tls_vminor == TLS_MINOR_VER_THREE); + if (tls13) + vminor = TLS_MINOR_VER_TWO; + else + vminor = tls->params.tls_vminor; for (;;) { /* Is there enough queued for a TLS header? */ if (sb->sb_tlscc < tls->params.tls_hlen) @@ -1852,7 +1901,9 @@ ktls_decrypt(struct socket *so) tls_len = sizeof(*hdr) + ntohs(hdr->tls_length); if (hdr->tls_vmajor != tls->params.tls_vmajor || - hdr->tls_vminor != tls->params.tls_vminor) + hdr->tls_vminor != vminor) + error = EINVAL; + else if (tls13 && hdr->tls_type != TLS_RLTYPE_APP) error = EINVAL; else if (tls_len < tls->params.tls_hlen || tls_len > tls->params.tls_hlen + TLS_MAX_MSG_SIZE_V10_2 + @@ -1895,6 +1946,13 @@ ktls_decrypt(struct socket *so) SOCKBUF_UNLOCK(sb); error = tls->sw_decrypt(tls, hdr, data, seqno, &trail_len); + if (error == 0) { + if (tls13) + error = tls13_find_record_type(tls, data, + tls_len, &trail_len, &record_type); + else + record_type = hdr->tls_type; + } if (error) { counter_u64_add(ktls_offload_failed_crypto, 1); @@ -1927,7 +1985,7 @@ ktls_decrypt(struct socket *so) } /* Allocate the control mbuf. */ - tgr.tls_type = hdr->tls_type; + tgr.tls_type = record_type; tgr.tls_vmajor = hdr->tls_vmajor; tgr.tls_vminor = hdr->tls_vminor; tgr.tls_length = htobe16(tls_len - tls->params.tls_hlen - diff --git a/sys/opencrypto/ktls_ocf.c b/sys/opencrypto/ktls_ocf.c index cbd1af6c6341..5fafb19a9c4e 100644 --- a/sys/opencrypto/ktls_ocf.c +++ b/sys/opencrypto/ktls_ocf.c @@ -101,11 +101,21 @@ SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls12_chacha20_encrypts, CTLFLAG_RD, &ocf_tls12_chacha20_encrypts, "Total number of OCF TLS 1.2 Chacha20-Poly1305 encryption operations"); +static COUNTER_U64_DEFINE_EARLY(ocf_tls13_gcm_decrypts); +SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls13_gcm_decrypts, + CTLFLAG_RD, &ocf_tls13_gcm_decrypts, + "Total number of OCF TLS 1.3 GCM decryption operations"); + static COUNTER_U64_DEFINE_EARLY(ocf_tls13_gcm_encrypts); SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls13_gcm_encrypts, CTLFLAG_RD, &ocf_tls13_gcm_encrypts, "Total number of OCF TLS 1.3 GCM encryption operations"); +static COUNTER_U64_DEFINE_EARLY(ocf_tls13_chacha20_decrypts); +SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls13_chacha20_decrypts, + CTLFLAG_RD, &ocf_tls13_chacha20_decrypts, + "Total number of OCF TLS 1.3 Chacha20-Poly1305 decryption operations"); + static COUNTER_U64_DEFINE_EARLY(ocf_tls13_chacha20_encrypts); SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls13_chacha20_encrypts, CTLFLAG_RD, &ocf_tls13_chacha20_encrypts, @@ -599,6 +609,58 @@ ktls_ocf_tls13_aead_encrypt(struct ktls_session *tls, return (error); } +static int +ktls_ocf_tls13_aead_decrypt(struct ktls_session *tls, + const struct tls_record_layer *hdr, struct mbuf *m, uint64_t seqno, + int *trailer_len) +{ + struct tls_aead_data_13 ad; + struct cryptop crp; + struct ocf_session *os; + int error; + u_int tag_len; + + os = tls->cipher; + + tag_len = tls->params.tls_tlen - 1; + + /* Payload must contain at least one byte for the record type. */ + if (ntohs(hdr->tls_length) < tag_len + 1) + return (EBADMSG); + + crypto_initreq(&crp, os->sid); + + /* Setup the nonce. */ + memcpy(crp.crp_iv, tls->params.iv, tls->params.iv_len); + *(uint64_t *)(crp.crp_iv + 4) ^= htobe64(seqno); + + /* Setup the AAD. */ + ad.type = hdr->tls_type; + ad.tls_vmajor = hdr->tls_vmajor; + ad.tls_vminor = hdr->tls_vminor; + ad.tls_length = hdr->tls_length; + crp.crp_aad = &ad; + crp.crp_aad_length = sizeof(ad); + + crp.crp_payload_start = tls->params.tls_hlen; + crp.crp_payload_length = ntohs(hdr->tls_length) - tag_len; + crp.crp_digest_start = crp.crp_payload_start + crp.crp_payload_length; + + crp.crp_op = CRYPTO_OP_DECRYPT | CRYPTO_OP_VERIFY_DIGEST; + crp.crp_flags = CRYPTO_F_CBIMM | CRYPTO_F_IV_SEPARATE; + crypto_use_mbuf(&crp, m); + + if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) + counter_u64_add(ocf_tls13_gcm_decrypts, 1); + else + counter_u64_add(ocf_tls13_chacha20_decrypts, 1); + error = ktls_ocf_dispatch(os, &crp); + + crypto_destroyreq(&crp); + *trailer_len = tag_len; + return (error); +} + static void ktls_ocf_free(struct ktls_session *tls) { @@ -638,11 +700,6 @@ ktls_ocf_try(struct socket *so, struct ktls_session *tls, int direction) tls->params.tls_vminor > TLS_MINOR_VER_THREE) return (EPROTONOSUPPORT); - /* TLS 1.3 is not yet supported for receive. */ - if (direction == KTLS_RX && - tls->params.tls_vminor == TLS_MINOR_VER_THREE) - return (EPROTONOSUPPORT); - csp.csp_flags |= CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD; csp.csp_mode = CSP_MODE_AEAD; csp.csp_cipher_alg = CRYPTO_AES_NIST_GCM_16; @@ -710,11 +767,6 @@ ktls_ocf_try(struct socket *so, struct ktls_session *tls, int direction) tls->params.tls_vminor > TLS_MINOR_VER_THREE) return (EPROTONOSUPPORT); - /* TLS 1.3 is not yet supported for receive. */ - if (direction == KTLS_RX && - tls->params.tls_vminor == TLS_MINOR_VER_THREE) - return (EPROTONOSUPPORT); - csp.csp_flags |= CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD; csp.csp_mode = CSP_MODE_AEAD; csp.csp_cipher_alg = CRYPTO_CHACHA20_POLY1305; @@ -758,7 +810,10 @@ ktls_ocf_try(struct socket *so, struct ktls_session *tls, int direction) else tls->sw_encrypt = ktls_ocf_tls12_aead_encrypt; } else { - tls->sw_decrypt = ktls_ocf_tls12_aead_decrypt; + if (tls->params.tls_vminor == TLS_MINOR_VER_THREE) + tls->sw_decrypt = ktls_ocf_tls13_aead_decrypt; + else + tls->sw_decrypt = ktls_ocf_tls12_aead_decrypt; } } else { tls->sw_encrypt = ktls_ocf_tls_cbc_encrypt; diff --git a/tests/sys/kern/ktls_test.c b/tests/sys/kern/ktls_test.c index a994f675711b..759a23455f25 100644 --- a/tests/sys/kern/ktls_test.c +++ b/tests/sys/kern/ktls_test.c @@ -816,26 +816,74 @@ encrypt_tls_12_aead(struct tls_enable *en, uint8_t record_type, uint64_t seqno, return (record_len); } +static size_t +encrypt_tls_13_aead(struct tls_enable *en, uint8_t record_type, uint64_t seqno, + const void *src, size_t len, void *dst, size_t padding) +{ + struct tls_record_layer *hdr; + struct tls_aead_data_13 aad; + char nonce[12]; + char *buf; + size_t hdr_len, mac_len, record_len; + + hdr = dst; + + hdr_len = tls_header_len(en); + mac_len = tls_mac_len(en); + record_len = hdr_len + len + 1 + padding + mac_len; + + hdr->tls_type = TLS_RLTYPE_APP; + hdr->tls_vmajor = TLS_MAJOR_VER_ONE; + hdr->tls_vminor = TLS_MINOR_VER_TWO; + hdr->tls_length = htons(record_len - sizeof(*hdr)); + + tls_13_aad(en, hdr, seqno, &aad); + tls_13_nonce(en, seqno, nonce); + + /* + * Have to use a temporary buffer for the input so that the record + * type can be appended. + */ + buf = malloc(len + 1 + padding); + memcpy(buf, src, len); + buf[len] = record_type; + memset(buf + len + 1, 0, padding); + + ATF_REQUIRE(aead_encrypt(tls_EVP_CIPHER(en), en->cipher_key, nonce, + &aad, sizeof(aad), buf, (char *)dst + hdr_len, len + 1 + padding, + (char *)dst + hdr_len + len + 1 + padding, mac_len)); + + free(buf); + + return (record_len); +} + static size_t encrypt_tls_aead(struct tls_enable *en, uint8_t record_type, uint64_t seqno, - const void *src, size_t len, void *dst, size_t avail) + const void *src, size_t len, void *dst, size_t avail, size_t padding) { size_t record_len; - record_len = tls_header_len(en) + len + tls_trailer_len(en); + record_len = tls_header_len(en) + len + padding + tls_trailer_len(en); ATF_REQUIRE(record_len <= avail); - ATF_REQUIRE(encrypt_tls_12_aead(en, record_type, seqno, src, len, - dst) == record_len); + if (en->tls_vminor == TLS_MINOR_VER_TWO) { + ATF_REQUIRE(padding == 0); + ATF_REQUIRE(encrypt_tls_12_aead(en, record_type, seqno, src, + len, dst) == record_len); + } else + ATF_REQUIRE(encrypt_tls_13_aead(en, record_type, seqno, src, + len, dst, padding) == record_len); return (record_len); } static size_t encrypt_tls_record(struct tls_enable *en, uint8_t record_type, uint64_t seqno, - const void *src, size_t len, void *dst, size_t avail) + const void *src, size_t len, void *dst, size_t avail, size_t padding) { - return (encrypt_tls_aead(en, record_type, seqno, src, len, dst, avail)); + return (encrypt_tls_aead(en, record_type, seqno, src, len, dst, avail, + padding)); } static void @@ -1132,14 +1180,19 @@ ktls_receive_tls_record(struct tls_enable *en, int fd, uint8_t record_type, tgr = (struct tls_get_record *)CMSG_DATA(cmsg); ATF_REQUIRE(tgr->tls_type == record_type); ATF_REQUIRE(tgr->tls_vmajor == en->tls_vmajor); - ATF_REQUIRE(tgr->tls_vminor == en->tls_vminor); + /* XXX: Not sure if this is what OpenSSL expects? */ + if (en->tls_vminor == TLS_MINOR_VER_THREE) + ATF_REQUIRE(tgr->tls_vminor == TLS_MINOR_VER_TWO); + else + ATF_REQUIRE(tgr->tls_vminor == en->tls_vminor); ATF_REQUIRE(tgr->tls_length == htons(rv)); return (rv); } static void -test_ktls_receive_app_data(struct tls_enable *en, uint64_t seqno, size_t len) +test_ktls_receive_app_data(struct tls_enable *en, uint64_t seqno, size_t len, + size_t padding) { struct kevent ev; char *plaintext, *received, *outbuf; @@ -1180,11 +1233,11 @@ test_ktls_receive_app_data(struct tls_enable *en, uint64_t seqno, size_t len) if (outbuf_len == 0) { ATF_REQUIRE(written < len); todo = len - written; - if (todo > TLS_MAX_MSG_SIZE_V10_2) - todo = TLS_MAX_MSG_SIZE_V10_2; + if (todo > TLS_MAX_MSG_SIZE_V10_2 - padding) + todo = TLS_MAX_MSG_SIZE_V10_2 - padding; outbuf_len = encrypt_tls_record(en, TLS_RLTYPE_APP, seqno, plaintext + written, - todo, outbuf, outbuf_cap); + todo, outbuf, outbuf_cap, padding); outbuf_sent = 0; written += todo; seqno++; @@ -1240,14 +1293,6 @@ test_ktls_receive_app_data(struct tls_enable *en, uint64_t seqno, size_t len) M(aes256_cbc_1_0_sha1, CRYPTO_AES_CBC, 256 / 8, \ CRYPTO_SHA1_HMAC) -#define TLS_12_TESTS(M) \ - M(aes128_gcm_1_2, CRYPTO_AES_NIST_GCM_16, 128 / 8, 0, \ - TLS_MINOR_VER_TWO) \ - M(aes256_gcm_1_2, CRYPTO_AES_NIST_GCM_16, 256 / 8, 0, \ - TLS_MINOR_VER_TWO) \ - M(chacha20_poly1305_1_2, CRYPTO_CHACHA20_POLY1305, 256 / 8, 0, \ - TLS_MINOR_VER_TWO) - #define TLS_13_TESTS(M) \ M(aes128_gcm_1_3, CRYPTO_AES_NIST_GCM_16, 128 / 8, 0, \ TLS_MINOR_VER_THREE) \ @@ -1541,7 +1586,7 @@ ATF_TC_BODY(ktls_transmit_invalid_##name, tc) \ INVALID_CIPHER_SUITES(GEN_INVALID_TRANSMIT_TEST); #define GEN_RECEIVE_APP_DATA_TEST(cipher_name, cipher_alg, key_size, \ - auth_alg, minor, name, len) \ + auth_alg, minor, name, len, padding) \ ATF_TC_WITHOUT_HEAD(ktls_receive_##cipher_name##_##name); \ ATF_TC_BODY(ktls_receive_##cipher_name##_##name, tc) \ { \ @@ -1552,7 +1597,7 @@ ATF_TC_BODY(ktls_receive_##cipher_name##_##name, tc) \ seqno = random(); \ build_tls_enable(cipher_alg, key_size, auth_alg, minor, seqno, \ &en); \ - test_ktls_receive_app_data(&en, seqno, len); \ + test_ktls_receive_app_data(&en, seqno, len, padding); \ free_tls_enable(&en); \ } @@ -1563,9 +1608,9 @@ ATF_TC_BODY(ktls_receive_##cipher_name##_##name, tc) \ #define GEN_RECEIVE_TESTS(cipher_name, cipher_alg, key_size, auth_alg, \ minor) \ GEN_RECEIVE_APP_DATA_TEST(cipher_name, cipher_alg, key_size, \ - auth_alg, minor, short, 64) \ + auth_alg, minor, short, 64, 0) \ GEN_RECEIVE_APP_DATA_TEST(cipher_name, cipher_alg, key_size, \ - auth_alg, minor, long, 64 * 1024) + auth_alg, minor, long, 64 * 1024, 0) #define ADD_RECEIVE_TESTS(cipher_name, cipher_alg, key_size, auth_alg, \ minor) \ @@ -1586,7 +1631,28 @@ ATF_TC_BODY(ktls_receive_##cipher_name##_##name, tc) \ * Note that receive is currently only supported for TLS 1.2 AEAD * cipher suites. */ -TLS_12_TESTS(GEN_RECEIVE_TESTS); +AES_GCM_TESTS(GEN_RECEIVE_TESTS); +CHACHA20_TESTS(GEN_RECEIVE_TESTS); + +#define GEN_PADDING_RECEIVE_TESTS(cipher_name, cipher_alg, key_size, \ + auth_alg, minor) \ + GEN_RECEIVE_APP_DATA_TEST(cipher_name, cipher_alg, key_size, \ + auth_alg, minor, short_padded, 64, 16) \ + GEN_RECEIVE_APP_DATA_TEST(cipher_name, cipher_alg, key_size, \ + auth_alg, minor, long_padded, 64 * 1024, 15) + +#define ADD_PADDING_RECEIVE_TESTS(cipher_name, cipher_alg, key_size, \ + auth_alg, minor) \ + ADD_RECEIVE_APP_DATA_TEST(cipher_name, cipher_alg, key_size, \ + auth_alg, minor, short_padded) \ + ADD_RECEIVE_APP_DATA_TEST(cipher_name, cipher_alg, key_size, \ + auth_alg, minor, long_padded) + +/* + * For TLS 1.3 cipher suites, run two additional receive tests which + * use add padding to each record. + */ +TLS_13_TESTS(GEN_PADDING_RECEIVE_TESTS); static void test_ktls_invalid_receive_cipher_suite(struct tls_enable *en) @@ -1597,12 +1663,7 @@ test_ktls_invalid_receive_cipher_suite(struct tls_enable *en) ATF_REQUIRE(setsockopt(sockets[1], IPPROTO_TCP, TCP_RXTLS_ENABLE, en, sizeof(*en)) == -1); - - /* - * XXX: TLS 1.3 fails with ENOTSUP before checking for invalid - * ciphers. - */ - ATF_REQUIRE(errno == EINVAL || errno == ENOTSUP); + ATF_REQUIRE(errno == EINVAL); ATF_REQUIRE(close(sockets[1]) == 0); ATF_REQUIRE(close(sockets[0]) == 0); @@ -1642,7 +1703,7 @@ test_ktls_unsupported_receive_cipher_suite(struct tls_enable *en) ATF_REQUIRE(setsockopt(sockets[1], IPPROTO_TCP, TCP_RXTLS_ENABLE, en, sizeof(*en)) == -1); - ATF_REQUIRE(errno == EPROTONOSUPPORT || errno == ENOTSUP); + ATF_REQUIRE(errno == EPROTONOSUPPORT); ATF_REQUIRE(close(sockets[1]) == 0); ATF_REQUIRE(close(sockets[0]) == 0); @@ -1673,7 +1734,6 @@ ATF_TC_BODY(ktls_receive_unsupported_##name, tc) \ * rejected. */ AES_CBC_TESTS(GEN_UNSUPPORTED_RECEIVE_TEST); -TLS_13_TESTS(GEN_UNSUPPORTED_RECEIVE_TEST); /* * Try to perform an invalid sendto(2) on a TXTLS-enabled socket, to exercise @@ -1728,8 +1788,9 @@ ATF_TP_ADD_TCS(tp) /* Receive tests */ AES_CBC_TESTS(ADD_UNSUPPORTED_RECEIVE_TEST); - TLS_12_TESTS(ADD_RECEIVE_TESTS); - TLS_13_TESTS(ADD_UNSUPPORTED_RECEIVE_TEST); + AES_GCM_TESTS(ADD_RECEIVE_TESTS); + CHACHA20_TESTS(ADD_RECEIVE_TESTS); + TLS_13_TESTS(ADD_PADDING_RECEIVE_TESTS); INVALID_CIPHER_SUITES(ADD_INVALID_RECEIVE_TEST); /* Miscellaneous */
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?202204292312.23TNC9H3044263>