Date: Thu, 14 May 2020 21:46:12 +0000 (UTC) From: Rick Macklem <rmacklem@FreeBSD.org> To: src-committers@freebsd.org, svn-src-projects@freebsd.org Subject: svn commit: r361062 - in projects/nfs-over-tls: sys/kern sys/rpc sys/rpc/rpcsec_tls usr.sbin/rpctlscd usr.sbin/rpctlssd Message-ID: <202005142146.04ELkCmN027147@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: rmacklem Date: Thu May 14 21:46:12 2020 New Revision: 361062 URL: https://svnweb.freebsd.org/changeset/base/361062 Log: Add code that does upcalls to the daemons when an non-application data record is at the top of the socket receive queue. Define a new socket flag MSG_TLSAPPDATA which tells soreceive_generic() to return ENXIO if the first record in the receive queue is not an application data record. ENXIO then triggers svc_vc_recv() to do an upcall to the rpctlssd daemon, so the daemon can do a SSL_read() for 0 bytes to handle the record. This patch adds some similar code to the client side rpctlscd daemon, but the kernel code to do the upcall still needs to be written. For the client side, I do not currently know how to test it, because it is in disconnect when the "close alert" is sent via SSL_shutdown(). Testing of this client code may need to wait until TLS1.3 support is in the KERN_TLS. Modified: projects/nfs-over-tls/sys/kern/uipc_socket.c projects/nfs-over-tls/sys/rpc/rpcsec_tls.h projects/nfs-over-tls/sys/rpc/rpcsec_tls/rpctls_impl.c projects/nfs-over-tls/sys/rpc/rpcsec_tls/rpctlscd.x projects/nfs-over-tls/sys/rpc/rpcsec_tls/rpctlssd.x projects/nfs-over-tls/sys/rpc/svc_vc.c projects/nfs-over-tls/usr.sbin/rpctlscd/rpctlscd.c projects/nfs-over-tls/usr.sbin/rpctlssd/rpctlssd.c Modified: projects/nfs-over-tls/sys/kern/uipc_socket.c ============================================================================== --- projects/nfs-over-tls/sys/kern/uipc_socket.c Thu May 14 21:36:59 2020 (r361061) +++ projects/nfs-over-tls/sys/kern/uipc_socket.c Thu May 14 21:46:12 2020 (r361062) @@ -2052,6 +2052,31 @@ dontblock: if (m != NULL && m->m_type == MT_CONTROL) { struct mbuf *cm = NULL, *cmn; struct mbuf **cme = &cm; + struct cmsghdr *cmsg; + struct tls_get_record tgr; + + /* + * For MSG_TLSAPPDATA, check for a non-application data + * record. If found, then return ENXIO without removing + * it from the receive queue. This allows a subsequent + * call without MSG_TLSAPPDATA to receive it. + * Note that, for TLS, there should only be a single + * control mbuf with the TLS_GET_RECORD message in it. + */ + if (flags & MSG_TLSAPPDATA) { + cmsg = mtod(m, struct cmsghdr *); + if (cmsg->cmsg_type == TLS_GET_RECORD && + cmsg->cmsg_len == CMSG_LEN(sizeof(tgr))) { + memcpy(&tgr, CMSG_DATA(cmsg), sizeof(tgr)); + /* This will need to change for TLS 1.3. */ + if (tgr.tls_type != TLS_RLTYPE_APP) { + SOCKBUF_UNLOCK(&so->so_rcv); +printf("fnd non-app rec=%d\n", tgr.tls_type); + error = ENXIO; + goto release; + } + } + } do { if (flags & MSG_PEEK) { Modified: projects/nfs-over-tls/sys/rpc/rpcsec_tls.h ============================================================================== --- projects/nfs-over-tls/sys/rpc/rpcsec_tls.h Thu May 14 21:36:59 2020 (r361061) +++ projects/nfs-over-tls/sys/rpc/rpcsec_tls.h Thu May 14 21:46:12 2020 (r361062) @@ -47,6 +47,10 @@ /* Functions that perform upcalls to the rpctlsd daemon. */ enum clnt_stat rpctls_connect(CLIENT *newclient, struct socket *so, uint64_t *sslp); +enum clnt_stat rpctls_cl_handlerecord(uint64_t sec, uint64_t usec, + uint64_t ssl); +enum clnt_stat rpctls_srv_handlerecord(uint64_t sec, uint64_t usec, + uint64_t ssl); enum clnt_stat rpctls_cl_disconnect(uint64_t sec, uint64_t usec, uint64_t ssl); enum clnt_stat rpctls_srv_disconnect(uint64_t sec, uint64_t usec, uint64_t ssl); Modified: projects/nfs-over-tls/sys/rpc/rpcsec_tls/rpctls_impl.c ============================================================================== --- projects/nfs-over-tls/sys/rpc/rpcsec_tls/rpctls_impl.c Thu May 14 21:36:59 2020 (r361061) +++ projects/nfs-over-tls/sys/rpc/rpcsec_tls/rpctls_impl.c Thu May 14 21:46:12 2020 (r361062) @@ -385,6 +385,53 @@ printf("aft wakeup\n"); return (stat); } +/* Do an upcall to handle an non-application data record using TLS. */ +enum clnt_stat +rpctls_cl_handlerecord(uint64_t sec, uint64_t usec, uint64_t ssl) +{ + struct rpctlscd_handlerecord_arg arg; + enum clnt_stat stat; + CLIENT *cl; + +printf("In rpctls_cl_handlerecord\n"); + cl = rpctls_connect_client(); +printf("handlerecord_client=%p\n", cl); + if (cl == NULL) + return (RPC_FAILED); + + /* Do the handlerecord upcall. */ + arg.sec = sec; + arg.usec = usec; + arg.ssl = ssl; + stat = rpctlscd_handlerecord_1(&arg, NULL, cl); +printf("aft handlerecord upcall=%d\n", stat); + CLNT_RELEASE(cl); + return (stat); +} + +enum clnt_stat +rpctls_srv_handlerecord(uint64_t sec, uint64_t usec, uint64_t ssl) +{ + struct rpctlssd_handlerecord_arg arg; + enum clnt_stat stat; + CLIENT *cl; + +printf("In rpctls_srv_handlerecord\n"); + cl = rpctls_server_client(); +printf("srv handlerecord_client=%p\n", cl); + if (cl == NULL) + return (RPC_FAILED); + + /* Do the handlerecord upcall. */ + arg.sec = sec; + arg.usec = usec; + arg.ssl = ssl; + stat = rpctlssd_handlerecord_1(&arg, NULL, cl); +printf("aft srv handlerecord upcall=%d\n", stat); + CLNT_RELEASE(cl); + return (stat); +} + /* Do an upcall to shut down a socket using TLS. */ enum clnt_stat rpctls_cl_disconnect(uint64_t sec, uint64_t usec, uint64_t ssl) Modified: projects/nfs-over-tls/sys/rpc/rpcsec_tls/rpctlscd.x ============================================================================== --- projects/nfs-over-tls/sys/rpc/rpcsec_tls/rpctlscd.x Thu May 14 21:36:59 2020 (r361061) +++ projects/nfs-over-tls/sys/rpc/rpcsec_tls/rpctlscd.x Thu May 14 21:46:12 2020 (r361062) @@ -35,6 +35,12 @@ struct rpctlscd_connect_res { uint64_t ssl; }; +struct rpctlscd_handlerecord_arg { + uint64_t sec; + uint64_t usec; + uint64_t ssl; +}; + struct rpctlscd_disconnect_arg { uint64_t sec; uint64_t usec; @@ -48,6 +54,8 @@ program RPCTLSCD { rpctlscd_connect_res RPCTLSCD_CONNECT(void) = 1; - void RPCTLSCD_DISCONNECT(rpctlscd_disconnect_arg) = 2; + void RPCTLSCD_HANDLERECORD(rpctlscd_handlerecord_arg) = 2; + + void RPCTLSCD_DISCONNECT(rpctlscd_disconnect_arg) = 3; } = 1; } = 0x40677374; Modified: projects/nfs-over-tls/sys/rpc/rpcsec_tls/rpctlssd.x ============================================================================== --- projects/nfs-over-tls/sys/rpc/rpcsec_tls/rpctlssd.x Thu May 14 21:36:59 2020 (r361061) +++ projects/nfs-over-tls/sys/rpc/rpcsec_tls/rpctlssd.x Thu May 14 21:46:12 2020 (r361062) @@ -38,6 +38,12 @@ struct rpctlssd_connect_res { uint32_t gid<>; }; +struct rpctlssd_handlerecord_arg { + uint64_t sec; + uint64_t usec; + uint64_t ssl; +}; + struct rpctlssd_disconnect_arg { uint64_t sec; uint64_t usec; @@ -51,6 +57,8 @@ program RPCTLSSD { rpctlssd_connect_res RPCTLSSD_CONNECT(void) = 1; - void RPCTLSSD_DISCONNECT(rpctlssd_disconnect_arg) = 2; + void RPCTLSSD_HANDLERECORD(rpctlssd_handlerecord_arg) = 2; + + void RPCTLSSD_DISCONNECT(rpctlssd_disconnect_arg) = 3; } = 1; } = 0x40677375; Modified: projects/nfs-over-tls/sys/rpc/svc_vc.c ============================================================================== --- projects/nfs-over-tls/sys/rpc/svc_vc.c Thu May 14 21:36:59 2020 (r361061) +++ projects/nfs-over-tls/sys/rpc/svc_vc.c Thu May 14 21:46:12 2020 (r361062) @@ -674,6 +674,7 @@ svc_vc_recv(SVCXPRT *xprt, struct rpc_msg *msg, uint32_t xid_plus_direction[2]; struct cmsghdr *cmsg; struct tls_get_record tgr; + enum clnt_stat ret; /* * Serialise access to the socket and our own record parsing @@ -748,6 +749,9 @@ svc_vc_recv(SVCXPRT *xprt, struct rpc_msg *msg, * If receiving is disabled so that a TLS handshake can be * done by the rpctlssd daemon, return FALSE here. */ + rcvflag = MSG_DONTWAIT; + if ((xprt->xp_tls & RPCTLS_FLAGS_HANDSHAKE) != 0) + rcvflag |= MSG_TLSAPPDATA; tryagain: if (xprt->xp_dontrcv) { sx_xunlock(&xprt->xp_lock); @@ -765,7 +769,6 @@ tryagain: uio.uio_resid = 1000000000; uio.uio_td = curthread; ctrl = m = NULL; - rcvflag = MSG_DONTWAIT; error = soreceive(so, NULL, &uio, &m, &ctrl, &rcvflag); if (error == EWOULDBLOCK) { @@ -784,6 +787,36 @@ tryagain: return (FALSE); } + /* + * A return of ENXIO indicates that there is a + * non-application data record at the head of the + * socket's receive queue, for TLS connections. + * This record needs to be handled in userland + * via an SSL_read() call, so do an upcall to the daemon. + */ + if ((xprt->xp_tls & RPCTLS_FLAGS_HANDSHAKE) != 0 && + error == ENXIO) { + /* Disable reception. */ + xprt->xp_dontrcv = TRUE; + sx_xunlock(&xprt->xp_lock); +printf("Call rpctls_srv_handlerecord\n"); + ret = rpctls_srv_handlerecord(xprt->xp_sslsec, + xprt->xp_sslusec, xprt->xp_sslrefno); + sx_xlock(&xprt->xp_lock); + xprt->xp_dontrcv = FALSE; + if (ret != RPC_SUCCESS) { + /* + * All we can do is soreceive() it and + * then toss it. + */ + rcvflag = MSG_DONTWAIT; + goto tryagain; + } + sx_xunlock(&xprt->xp_lock); + xprt_active(xprt); /* Harmless if already active. */ + return (FALSE); + } + if (error) { SOCKBUF_LOCK(&so->so_rcv); if (xprt->xp_upcallset) { @@ -815,15 +848,16 @@ if (ctrl->m_next != NULL) printf("EEK! svc list of con cmsg->cmsg_len == CMSG_LEN(sizeof(tgr))) { memcpy(&tgr, CMSG_DATA(cmsg), sizeof(tgr)); /* - * For now, just toss non-application - * data records. - * In the future, there may need to be - * an upcall done to the daemon. + * This should have been handled by + * the rpctls_svc_handlerecord() + * upcall. If not, all we can do is + * toss it away. */ if (tgr.tls_type != TLS_RLTYPE_APP) { printf("Got weird type=%d\n", tgr.tls_type); m_freem(m); m_free(ctrl); + rcvflag = MSG_DONTWAIT | MSG_TLSAPPDATA; goto tryagain; } } Modified: projects/nfs-over-tls/usr.sbin/rpctlscd/rpctlscd.c ============================================================================== --- projects/nfs-over-tls/usr.sbin/rpctlscd/rpctlscd.c Thu May 14 21:36:59 2020 (r361061) +++ projects/nfs-over-tls/usr.sbin/rpctlscd/rpctlscd.c Thu May 14 21:46:12 2020 (r361062) @@ -371,10 +371,47 @@ rpctlscd_verbose_out("rpctlsd_connect s=%d\n", s); } bool_t +rpctlscd_handlerecord_1_svc(struct rpctlscd_handlerecord_arg *argp, + void *result, struct svc_req *rqstp) +{ + struct ssl_entry *slp; + int ret; + char junk; + + slp = NULL; + if (argp->sec == rpctls_ssl_sec && argp->usec == + rpctls_ssl_usec) { + LIST_FOREACH(slp, &rpctls_ssllist, next) { + if (slp->refno == argp->ssl) + break; + } + } + + if (slp != NULL) { + rpctlscd_verbose_out("rpctlscd_handlerecord fd=%d\n", + slp->s); + /* + * An SSL_read() of 0 bytes should fail, but it should + * handle the non-application data record before doing so. + */ + ret = SSL_read(slp->ssl, &junk, 0); + if (ret > 0) { + if (rpctls_debug_level == 0) + syslog(LOG_ERR, "SSL_read returned %d", ret); + else + fprintf(stderr, "SSL_read returned %d\n", ret); + } + } else + return (FALSE); + return (TRUE); +} + +bool_t rpctlscd_disconnect_1_svc(struct rpctlscd_disconnect_arg *argp, void *result, struct svc_req *rqstp) { struct ssl_entry *slp; + int ret; slp = NULL; if (argp->sec == rpctls_ssl_sec && argp->usec == @@ -389,6 +426,13 @@ rpctlscd_disconnect_1_svc(struct rpctlscd_disconnect_a rpctlscd_verbose_out("rpctlscd_disconnect: fd=%d closed\n", slp->s); LIST_REMOVE(slp, next); + SSL_shutdown(slp->ssl); + /* Check to see if the peer has sent a close alert. */ + ret = SSL_get_shutdown(slp->ssl); +rpctlscd_verbose_out("get_shutdown=%d\n", ret); + if ((ret & (SSL_SENT_SHUTDOWN | SSL_RECEIVED_SHUTDOWN)) == + SSL_SENT_SHUTDOWN) + SSL_shutdown(slp->ssl); SSL_free(slp->ssl); /* * For RPC-over-TLS, this upcall is expected Modified: projects/nfs-over-tls/usr.sbin/rpctlssd/rpctlssd.c ============================================================================== --- projects/nfs-over-tls/usr.sbin/rpctlssd/rpctlssd.c Thu May 14 21:36:59 2020 (r361061) +++ projects/nfs-over-tls/usr.sbin/rpctlssd/rpctlssd.c Thu May 14 21:46:12 2020 (r361062) @@ -419,6 +419,49 @@ rpctlssd_verbose_out("rpctlsd_connect_svc s=%d\n", s); } bool_t +rpctlssd_handlerecord_1_svc(struct rpctlssd_handlerecord_arg *argp, + void *result, struct svc_req *rqstp) +{ + struct ssl_entry *slp; + int ret; + char junk; + + slp = NULL; + if (argp->sec == rpctls_ssl_sec && argp->usec == + rpctls_ssl_usec) { + LIST_FOREACH(slp, &rpctls_ssllist, next) { + if (slp->refno == argp->ssl) + break; + } + } + + if (slp != NULL) { + rpctlssd_verbose_out("rpctlssd_handlerecord fd=%d\n", + slp->s); + /* + * An SSL_read() of 0 bytes should fail, but it should + * handle the non-application data record before doing so. + */ + ret = SSL_read(slp->ssl, &junk, 0); + if (ret <= 0) { + /* Check to see if this was a close alert. */ + ret = SSL_get_shutdown(slp->ssl); +rpctlssd_verbose_out("get_shutdown=%d\n", ret); + if ((ret & (SSL_SENT_SHUTDOWN | + SSL_RECEIVED_SHUTDOWN)) == SSL_RECEIVED_SHUTDOWN) + SSL_shutdown(slp->ssl); + } else { + if (rpctls_debug_level == 0) + syslog(LOG_ERR, "SSL_read returned %d", ret); + else + fprintf(stderr, "SSL_read returned %d\n", ret); + } + } else + return (FALSE); + return (TRUE); +} + +bool_t rpctlssd_disconnect_1_svc(struct rpctlssd_disconnect_arg *argp, void *result, struct svc_req *rqstp) { @@ -442,6 +485,7 @@ rpctlssd_disconnect_1_svc(struct rpctlssd_disconnect_a * For RPC-over-TLS, this upcall is expected * to close off the socket. */ + shutdown(slp->s, SHUT_WR); close(slp->s); free(slp); } else @@ -679,7 +723,7 @@ rpctlssd_verbose_out("%s\n", cp2); } if (ret == 0) { if (rpctls_debug_level == 0) - syslog(LOG_ERR, "ktls not working\n"); + syslog(LOG_ERR, "ktls not working"); else fprintf(stderr, "ktls not working\n"); /*
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?202005142146.04ELkCmN027147>