Date: Sun, 8 Mar 2020 18:37:04 +0000 (UTC) From: Rick Macklem <rmacklem@FreeBSD.org> To: src-committers@freebsd.org, svn-src-projects@freebsd.org Subject: svn commit: r358765 - projects/nfs-over-tls/sys/rpc Message-ID: <202003081837.028Ib4Eg018199@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: rmacklem Date: Sun Mar 8 18:37:04 2020 New Revision: 358765 URL: https://svnweb.freebsd.org/changeset/base/358765 Log: Make a bunch of changes to the kernel RPC so that it can handle reception of ext_pgs mbuf lists. jhb@ thinks this will be be needed for certain cases of the KERN_TLS. There are also some changes for handling of flags passed down from the rpctlssd daemon that indicate the results of client certificate validation. One of these flags, RPCTLS_FLAGS_DISABLE, causes all RPCs on the connection to fail with AUTH_REJECTEDCRED. The others will be used by future commits to the NFS server code to check against new export flags. There are also changes in rpcsec_tls and the nfs code to make this support work. Modified: projects/nfs-over-tls/sys/rpc/clnt.h projects/nfs-over-tls/sys/rpc/clnt_bck.c projects/nfs-over-tls/sys/rpc/clnt_vc.c projects/nfs-over-tls/sys/rpc/rpc_generic.c projects/nfs-over-tls/sys/rpc/rpcsec_tls.h projects/nfs-over-tls/sys/rpc/svc.c projects/nfs-over-tls/sys/rpc/svc.h projects/nfs-over-tls/sys/rpc/svc_auth.c projects/nfs-over-tls/sys/rpc/svc_vc.c Modified: projects/nfs-over-tls/sys/rpc/clnt.h ============================================================================== --- projects/nfs-over-tls/sys/rpc/clnt.h Sun Mar 8 18:24:15 2020 (r358764) +++ projects/nfs-over-tls/sys/rpc/clnt.h Sun Mar 8 18:37:04 2020 (r358765) @@ -120,6 +120,7 @@ struct rpc_callextra { void *rc_feedback_arg; /* argument for callback */ struct rpc_timers *rc_timers; /* optional RTT timers */ struct rpc_err rc_err; /* detailed call status */ + u_int rc_mbufoffs; /* Offset in resultsp mbuf */ }; #endif Modified: projects/nfs-over-tls/sys/rpc/clnt_bck.c ============================================================================== --- projects/nfs-over-tls/sys/rpc/clnt_bck.c Sun Mar 8 18:24:15 2020 (r358764) +++ projects/nfs-over-tls/sys/rpc/clnt_bck.c Sun Mar 8 18:37:04 2020 (r358765) @@ -61,8 +61,11 @@ __FBSDID("$FreeBSD$"); * connection provided by the client to the server. */ +#include "opt_kern_tls.h" + #include <sys/param.h> #include <sys/systm.h> +#include <sys/ktls.h> #include <sys/lock.h> #include <sys/malloc.h> #include <sys/mbuf.h> @@ -84,7 +87,12 @@ __FBSDID("$FreeBSD$"); #include <rpc/rpc.h> #include <rpc/rpc_com.h> #include <rpc/krpc.h> +#include <rpc/rpcsec_tls.h> +#ifdef KERN_TLS +extern u_int ktls_maxlen; +#endif + struct cmessage { struct cmsghdr cmsg; struct cmsgcred cmcred; @@ -203,7 +211,8 @@ clnt_bck_call( uint32_t xid; struct mbuf *mreq = NULL, *results; struct ct_request *cr; - int error; + int error, maxextsiz; + uint32_t junk; cr = malloc(sizeof(struct ct_request), M_RPC, M_WAITOK); @@ -296,6 +305,18 @@ call_again: TAILQ_INSERT_TAIL(&ct->ct_pending, cr, cr_link); mtx_unlock(&ct->ct_lock); + /* For RPC-over-TLS, copy mrep to a chain of ext_pgs. */ + if ((xprt->xp_tls & RPCTLS_FLAGS_HANDSHAKE) != 0) { + /* + * Copy the mbuf chain to a chain of + * ext_pgs mbuf(s) as required by KERN_TLS. + */ + maxextsiz = TLS_MAX_MSG_SIZE_V10_2; +#ifdef KERN_TLS + maxextsiz = min(maxextsiz, ktls_maxlen); +#endif + mreq = _rpc_copym_into_ext_pgs(mreq, maxextsiz); + } /* * sosend consumes mreq. */ @@ -403,7 +424,9 @@ got_reply: ext->rc_feedback(FEEDBACK_OK, proc, ext->rc_feedback_arg); xdrmbuf_create(&xdrs, cr->cr_mrep, XDR_DECODE); - ok = xdr_replymsg(&xdrs, &reply_msg); + ok = xdr_uint32_t(&xdrs, &junk); + if (ok) + ok = xdr_replymsg(&xdrs, &reply_msg); cr->cr_mrep = NULL; if (ok) { @@ -422,6 +445,14 @@ got_reply: } else { KASSERT(results, ("auth validated but no result")); + if (ext) { + if ((results->m_flags & M_NOMAP) != + 0) + ext->rc_mbufoffs = + xdrs.x_handy; + else + ext->rc_mbufoffs = 0; + } *resultsp = results; } } /* end successful completion */ Modified: projects/nfs-over-tls/sys/rpc/clnt_vc.c ============================================================================== --- projects/nfs-over-tls/sys/rpc/clnt_vc.c Sun Mar 8 18:24:15 2020 (r358764) +++ projects/nfs-over-tls/sys/rpc/clnt_vc.c Sun Mar 8 18:37:04 2020 (r358765) @@ -84,6 +84,7 @@ __FBSDID("$FreeBSD$"); #include <rpc/rpc.h> #include <rpc/rpc_com.h> #include <rpc/krpc.h> +#include <rpc/rpcsec_tls.h> #ifdef KERN_TLS extern u_int ktls_maxlen; @@ -532,6 +533,19 @@ got_reply: if (ext && ext->rc_feedback) ext->rc_feedback(FEEDBACK_OK, proc, ext->rc_feedback_arg); +#ifdef notnow +{ struct mbuf *m, *m2; +int txxxx; +if (cr->cr_mrep != NULL) { +txxxx = m_length(cr->cr_mrep, NULL); +if (txxxx > 0) { +m = mb_copym_ext_pgs(cr->cr_mrep, txxxx, 16384, M_WAITOK, + false, mb_free_mext_pgs, &m2); +m2 = cr->cr_mrep; +cr->cr_mrep = m; +m_freem(m2); +} } } +#endif xdrmbuf_create(&xdrs, cr->cr_mrep, XDR_DECODE); ok = xdr_replymsg(&xdrs, &reply_msg); cr->cr_mrep = NULL; @@ -553,6 +567,14 @@ got_reply: } else { KASSERT(results, ("auth validated but no result")); + if (ext) { + if ((results->m_flags & M_NOMAP) != + 0) + ext->rc_mbufoffs = + xdrs.x_handy; + else + ext->rc_mbufoffs = 0; + } *resultsp = results; } } /* end successful completion */ @@ -749,8 +771,9 @@ clnt_vc_control(CLIENT *cl, u_int request, void *info) if (ct->ct_backchannelxprt == NULL) { xprt->xp_p2 = ct; if (ct->ct_tls) - xprt->xp_tls = TRUE; + xprt->xp_tls = RPCTLS_FLAGS_HANDSHAKE; ct->ct_backchannelxprt = xprt; +printf("backch tls=0x%x xprt=%p\n", xprt->xp_tls, xprt); } break; @@ -1032,9 +1055,11 @@ clnt_vc_soupcall(struct socket *so, void *arg, int wai ntohl(xid_plus_direction[1]); /* Check message direction. */ if (xid_plus_direction[1] == CALL) { +printf("Got backchannel callback\n"); /* This is a backchannel request. */ mtx_lock(&ct->ct_lock); xprt = ct->ct_backchannelxprt; +printf("backxprt=%p\n", xprt); if (xprt == NULL) { mtx_unlock(&ct->ct_lock); /* Just throw it away. */ Modified: projects/nfs-over-tls/sys/rpc/rpc_generic.c ============================================================================== --- projects/nfs-over-tls/sys/rpc/rpc_generic.c Sun Mar 8 18:24:15 2020 (r358764) +++ projects/nfs-over-tls/sys/rpc/rpc_generic.c Sun Mar 8 18:37:04 2020 (r358765) @@ -904,8 +904,7 @@ _rpc_copym_into_ext_pgs(struct mbuf *mp, int maxextsiz tlen = mp->m_len; m2 = mp; for (m = mp->m_next; m != NULL; m = m->m_next) { - if ((m->m_flags & (M_EXT | M_NOMAP)) == - (M_EXT | M_NOMAP)) + if ((m->m_flags & M_NOMAP) != 0) break; tlen += m->m_len; m2 = m; Modified: projects/nfs-over-tls/sys/rpc/rpcsec_tls.h ============================================================================== --- projects/nfs-over-tls/sys/rpc/rpcsec_tls.h Sun Mar 8 18:24:15 2020 (r358764) +++ projects/nfs-over-tls/sys/rpc/rpcsec_tls.h Sun Mar 8 18:37:04 2020 (r358765) @@ -35,10 +35,16 @@ #define RPCTLS_SYSC_CONNECT 2 #define RPCTLS_SYSC_SERVER 3 +/* Flag bits to indicate certificate results. */ +#define RPCTLS_FLAGS_HANDSHAKE 0x01 +#define RPCTLS_FLAGS_GOTCERT 0x02 +#define RPCTLS_FLAGS_SELFSIGNED 0x04 +#define RPCTLS_FLAGS_VERIFIED 0x08 +#define RPCTLS_FLAGS_DISABLED 0x10 + #ifdef _KERNEL /* Functions that perform upcalls to the rpctlsd daemon. */ enum clnt_stat rpctls_connect(CLIENT *newclient, struct socket *so); -enum clnt_stat rpctls_server(struct socket *so); /* String for AUTH_TLS reply verifier. */ #define RPCTLS_START_STRING "STARTTLS" Modified: projects/nfs-over-tls/sys/rpc/svc.c ============================================================================== --- projects/nfs-over-tls/sys/rpc/svc.c Sun Mar 8 18:24:15 2020 (r358764) +++ projects/nfs-over-tls/sys/rpc/svc.c Sun Mar 8 18:37:04 2020 (r358765) @@ -670,10 +670,13 @@ svc_sendreply_common(struct svc_req *rqstp, struct rpc replay_setreply(xprt->xp_pool->sp_rcache, rply, svc_getrpccaller(rqstp), body); +printf("sendreply_common\n"); if (!SVCAUTH_WRAP(&rqstp->rq_auth, &body)) return (FALSE); +printf("at SVC_REPLY\n"); ok = SVC_REPLY(xprt, rply, rqstp->rq_addr, body, &rqstp->rq_reply_seq); +printf("aft SVC_REPLY ok=%d\n", ok); if (rqstp->rq_addr) { free(rqstp->rq_addr, M_SONAME); rqstp->rq_addr = NULL; @@ -814,6 +817,7 @@ svcerr_auth(struct svc_req *rqstp, enum auth_stat why) if (xprt->xp_pool->sp_rcache) replay_setreply(xprt->xp_pool->sp_rcache, &rply, svc_getrpccaller(rqstp), NULL); +printf("SVC SENDAUTHERR\n"); svc_sendreply_common(rqstp, &rply, NULL); } @@ -976,6 +980,7 @@ svc_getreq(SVCXPRT *xprt, struct svc_req **rqstp_ret) * should not be dispatched to the * application. */ +printf("AUTH FAILED=%d\n", why); if (why != RPCSEC_GSS_NODISPATCH) svcerr_auth(r, why); goto call_done; Modified: projects/nfs-over-tls/sys/rpc/svc.h ============================================================================== --- projects/nfs-over-tls/sys/rpc/svc.h Sun Mar 8 18:24:15 2020 (r358764) +++ projects/nfs-over-tls/sys/rpc/svc.h Sun Mar 8 18:37:04 2020 (r358765) @@ -175,7 +175,9 @@ typedef struct __rpc_svcxprt { int xp_upcallset; /* socket upcall is set up */ uint32_t xp_snd_cnt; /* # of bytes to send to socket */ uint32_t xp_snt_cnt; /* # of bytes sent to socket */ + u_int xp_mbufoffs; /* Offset into ext_pgs mbuf */ bool_t xp_dontrcv; /* Do not receive on the socket */ + uint32_t xp_tls; /* RPC-over-TLS on socket */ #else int xp_fd; u_short xp_port; /* associated port number */ Modified: projects/nfs-over-tls/sys/rpc/svc_auth.c ============================================================================== --- projects/nfs-over-tls/sys/rpc/svc_auth.c Sun Mar 8 18:24:15 2020 (r358764) +++ projects/nfs-over-tls/sys/rpc/svc_auth.c Sun Mar 8 18:37:04 2020 (r358765) @@ -53,6 +53,7 @@ __FBSDID("$FreeBSD$"); #include <sys/ucred.h> #include <rpc/rpc.h> +#include <rpc/rpcsec_tls.h> static enum auth_stat (*_svcauth_rpcsec_gss)(struct svc_req *, struct rpc_msg *) = NULL; @@ -94,12 +95,18 @@ _authenticate(struct svc_req *rqst, struct rpc_msg *ms dummy = _svcauth_null(rqst, msg); return (dummy); case AUTH_SYS: + if ((rqst->rq_xprt->xp_tls & RPCTLS_FLAGS_DISABLED) != 0) + return (AUTH_REJECTEDCRED); dummy = _svcauth_unix(rqst, msg); return (dummy); case AUTH_SHORT: + if ((rqst->rq_xprt->xp_tls & RPCTLS_FLAGS_DISABLED) != 0) + return (AUTH_REJECTEDCRED); dummy = _svcauth_short(rqst, msg); return (dummy); case RPCSEC_GSS: + if ((rqst->rq_xprt->xp_tls & RPCTLS_FLAGS_DISABLED) != 0) + return (AUTH_REJECTEDCRED); if (!_svcauth_rpcsec_gss) return (AUTH_REJECTEDCRED); dummy = _svcauth_rpcsec_gss(rqst, msg); Modified: projects/nfs-over-tls/sys/rpc/svc_vc.c ============================================================================== --- projects/nfs-over-tls/sys/rpc/svc_vc.c Sun Mar 8 18:24:15 2020 (r358764) +++ projects/nfs-over-tls/sys/rpc/svc_vc.c Sun Mar 8 18:37:04 2020 (r358765) @@ -45,10 +45,13 @@ __FBSDID("$FreeBSD$"); * and a record/tcp stream. */ +#include "opt_kern_tls.h" + #include <sys/param.h> #include <sys/limits.h> #include <sys/lock.h> #include <sys/kernel.h> +#include <sys/ktls.h> #include <sys/malloc.h> #include <sys/mbuf.h> #include <sys/mutex.h> @@ -66,12 +69,17 @@ __FBSDID("$FreeBSD$"); #include <netinet/tcp.h> #include <rpc/rpc.h> +#include <rpc/rpcsec_tls.h> #include <rpc/krpc.h> #include <rpc/rpc_com.h> #include <security/mac/mac_framework.h> +#ifdef KERN_TLS +extern u_int ktls_maxlen; +#endif + static bool_t svc_vc_rendezvous_recv(SVCXPRT *, struct rpc_msg *, struct sockaddr **, struct mbuf **); static enum xprt_stat svc_vc_rendezvous_stat(SVCXPRT *); @@ -581,6 +589,30 @@ svc_vc_process_pending(SVCXPRT *xprt) struct socket *so = xprt->xp_socket; struct mbuf *m; +{ struct mbuf *m1, *m2, *m3, *m4; + int txxxx; + m3 = cd->mpending; + m4 = NULL; + while (m3 != NULL && (m3->m_flags & M_NOMAP) != 0) { + m4 = m3; + m3 = m3->m_next; + } + if (m3 != NULL) { + txxxx = m_length(m3, NULL); + if (txxxx > 0) { + m1 = mb_copym_ext_pgs(m3, txxxx, 16384, M_WAITOK, + false, mb_free_mext_pgs, &m2); + if (m4 != NULL) { + m4->m_next = m1; + m_freem(m3); + } else { + m2 = cd->mpending; + cd->mpending = m1; + m_freem(m2); + } + } + } +} /* * If cd->resid is non-zero, we have part of the * record already, otherwise we are expecting a record @@ -610,7 +642,7 @@ svc_vc_process_pending(SVCXPRT *xprt) header = ntohl(header); cd->eor = (header & 0x80000000) != 0; cd->resid = header & 0x7fffffff; - m_adj(cd->mpending, sizeof(uint32_t)); + cd->resid += sizeof(uint32_t); } /* @@ -623,10 +655,14 @@ svc_vc_process_pending(SVCXPRT *xprt) while (cd->mpending && cd->resid) { m = cd->mpending; if (cd->mpending->m_next - || cd->mpending->m_len > cd->resid) - cd->mpending = m_split(cd->mpending, - cd->resid, M_WAITOK); - else + || cd->mpending->m_len > cd->resid) { + if ((cd->mpending->m_flags & M_NOMAP) != 0) + cd->mpending = mb_splitatpos_ext( + cd->mpending, cd->resid, M_WAITOK); + else + cd->mpending = m_split(cd->mpending, + cd->resid, M_WAITOK); + } else cd->mpending = NULL; if (cd->mreq) m_last(cd->mreq)->m_next = m; @@ -660,7 +696,7 @@ svc_vc_recv(SVCXPRT *xprt, struct rpc_msg *msg, struct socket* so = xprt->xp_socket; XDR xdrs; int error, rcvflag; - uint32_t xid_plus_direction[2]; + uint32_t xid_plus_direction[3], junk; /* * Serialise access to the socket and our own record parsing @@ -691,15 +727,15 @@ svc_vc_recv(SVCXPRT *xprt, struct rpc_msg *msg, m_copydata(cd->mreq, 0, sizeof(xid_plus_direction), (char *)xid_plus_direction); - xid_plus_direction[0] = - ntohl(xid_plus_direction[0]); xid_plus_direction[1] = ntohl(xid_plus_direction[1]); + xid_plus_direction[2] = + ntohl(xid_plus_direction[2]); /* Check message direction. */ - if (xid_plus_direction[1] == REPLY) { + if (xid_plus_direction[2] == REPLY) { clnt_bck_svccall(xprt->xp_p2, cd->mreq, - xid_plus_direction[0]); + xid_plus_direction[1]); cd->mreq = NULL; continue; } @@ -719,13 +755,18 @@ svc_vc_recv(SVCXPRT *xprt, struct rpc_msg *msg, sx_xunlock(&xprt->xp_lock); - if (! xdr_callmsg(&xdrs, msg)) { + if (! xdr_uint32_t(&xdrs, &junk) || + ! xdr_callmsg(&xdrs, msg)) { XDR_DESTROY(&xdrs); return (FALSE); } *addrp = NULL; *mp = xdrmbuf_getall(&xdrs); + if (((*mp)->m_flags & M_NOMAP) != 0) + xprt->xp_mbufoffs = xdrs.x_handy; + else + xprt->xp_mbufoffs = 0; XDR_DESTROY(&xdrs); return (TRUE); @@ -827,13 +868,31 @@ svc_vc_backchannel_recv(SVCXPRT *xprt, struct rpc_msg mtx_unlock(&ct->ct_lock); sx_xunlock(&xprt->xp_lock); +printf("recv backch m=%p\n", m); +{ struct mbuf *m1, *m2; +int txxxx; +if (m != NULL) { +txxxx = m_length(m, NULL); +if (txxxx > 0) { +m1 = mb_copym_ext_pgs(m, txxxx, 16384, M_WAITOK, + false, mb_free_mext_pgs, &m2); +m2 = m; +m = m1; +m_freem(m2); +} } } xdrmbuf_create(&xdrs, m, XDR_DECODE); if (! xdr_callmsg(&xdrs, msg)) { +printf("recv backch callmsg failed\n"); XDR_DESTROY(&xdrs); return (FALSE); } *addrp = NULL; *mp = xdrmbuf_getall(&xdrs); + if (((*mp)->m_flags & M_NOMAP) != 0) + xprt->xp_mbufoffs = xdrs.x_handy; + else + xprt->xp_mbufoffs = 0; +printf("backch offs=%d\n", xprt->xp_mbufoffs); XDR_DESTROY(&xdrs); return (TRUE); } @@ -845,7 +904,7 @@ svc_vc_reply(SVCXPRT *xprt, struct rpc_msg *msg, XDR xdrs; struct mbuf *mrep; bool_t stat = TRUE; - int error, len; + int error, len, maxextsiz; /* * Leave space for record mark. @@ -875,7 +934,23 @@ svc_vc_reply(SVCXPRT *xprt, struct rpc_msg *msg, len = mrep->m_pkthdr.len; *mtod(mrep, uint32_t *) = htonl(0x80000000 | (len - sizeof(uint32_t))); + + /* For RPC-over-TLS, copy mrep to a chain of ext_pgs. */ + if ((xprt->xp_tls & RPCTLS_FLAGS_HANDSHAKE) != 0) { + /* + * Copy the mbuf chain to a chain of + * ext_pgs mbuf(s) as required by KERN_TLS. + */ + maxextsiz = TLS_MAX_MSG_SIZE_V10_2; +#ifdef KERN_TLS + maxextsiz = min(maxextsiz, ktls_maxlen); +#endif + mrep = _rpc_copym_into_ext_pgs(mrep, maxextsiz); + } atomic_add_32(&xprt->xp_snd_cnt, len); + /* + * sosend consumes mreq. + */ error = sosend(xprt->xp_socket, NULL, NULL, mrep, NULL, 0, curthread); if (!error) { @@ -902,7 +977,7 @@ svc_vc_backchannel_reply(SVCXPRT *xprt, struct rpc_msg XDR xdrs; struct mbuf *mrep; bool_t stat = TRUE; - int error; + int error, maxextsiz; /* * Leave space for record mark. @@ -932,6 +1007,19 @@ svc_vc_backchannel_reply(SVCXPRT *xprt, struct rpc_msg *mtod(mrep, uint32_t *) = htonl(0x80000000 | (mrep->m_pkthdr.len - sizeof(uint32_t))); + + /* For RPC-over-TLS, copy mrep to a chain of ext_pgs. */ + if ((xprt->xp_tls & RPCTLS_FLAGS_HANDSHAKE) != 0) { + /* + * Copy the mbuf chain to a chain of + * ext_pgs mbuf(s) as required by KERN_TLS. + */ + maxextsiz = TLS_MAX_MSG_SIZE_V10_2; +#ifdef KERN_TLS + maxextsiz = min(maxextsiz, ktls_maxlen); +#endif + mrep = _rpc_copym_into_ext_pgs(mrep, maxextsiz); + } sx_xlock(&xprt->xp_lock); ct = (struct ct_data *)xprt->xp_p2; if (ct != NULL)
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?202003081837.028Ib4Eg018199>