From owner-dev-commits-src-main@freebsd.org Wed Jan 20 16:03:45 2021 Return-Path: Delivered-To: dev-commits-src-main@mailman.nyi.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2610:1c1:1:606c::19:1]) by mailman.nyi.freebsd.org (Postfix) with ESMTP id 8B5054FAFB4; Wed, 20 Jan 2021 16:03:45 +0000 (UTC) (envelope-from git@FreeBSD.org) Received: from mxrelay.nyi.freebsd.org (mxrelay.nyi.freebsd.org [IPv6:2610:1c1:1:606c::19:3]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits) server-digest SHA256 client-signature RSA-PSS (4096 bits) client-digest SHA256) (Client CN "mxrelay.nyi.freebsd.org", Issuer "R3" (verified OK)) by mx1.freebsd.org (Postfix) with ESMTPS id 4DLVgj3VP8z3F1Q; Wed, 20 Jan 2021 16:03:45 +0000 (UTC) (envelope-from git@FreeBSD.org) Received: from gitrepo.freebsd.org (gitrepo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:5]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits) server-digest SHA256) (Client did not present a certificate) by mxrelay.nyi.freebsd.org (Postfix) with ESMTPS id 6AA721C0A5; Wed, 20 Jan 2021 16:03:45 +0000 (UTC) (envelope-from git@FreeBSD.org) Received: from gitrepo.freebsd.org ([127.0.1.44]) by gitrepo.freebsd.org (8.16.1/8.16.1) with ESMTP id 10KG3jq4024041; Wed, 20 Jan 2021 16:03:45 GMT (envelope-from git@gitrepo.freebsd.org) Received: (from git@localhost) by gitrepo.freebsd.org (8.16.1/8.16.1/Submit) id 10KG3jqM024040; Wed, 20 Jan 2021 16:03:45 GMT (envelope-from git) Date: Wed, 20 Jan 2021 16:03:45 GMT Message-Id: <202101201603.10KG3jqM024040@gitrepo.freebsd.org> To: src-committers@FreeBSD.org, dev-commits-src-all@FreeBSD.org, dev-commits-src-main@FreeBSD.org From: Alan Somers Subject: git: 2247f4894174 - main - aio: micro-optimize the lio_opcode assignments MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit X-Git-Committer: asomers X-Git-Repository: src X-Git-Refname: refs/heads/main X-Git-Reftype: branch X-Git-Commit: 2247f48941743cab420394b5ff0fc65ec8c69a99 Auto-Submitted: auto-generated X-BeenThere: dev-commits-src-main@freebsd.org X-Mailman-Version: 2.1.34 Precedence: list List-Id: Commit messages for the main branch of the src repository List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 20 Jan 2021 16:03:45 -0000 The branch main has been updated by asomers: URL: https://cgit.FreeBSD.org/src/commit/?id=2247f48941743cab420394b5ff0fc65ec8c69a99 commit 2247f48941743cab420394b5ff0fc65ec8c69a99 Author: Alan Somers AuthorDate: 2021-01-03 04:25:05 +0000 Commit: Alan Somers CommitDate: 2021-01-20 16:02:25 +0000 aio: micro-optimize the lio_opcode assignments This allows slightly more efficient opcode testing in-kernel. It is transparent to userland, except to applications that sneakily submit aio fsync or aio mlock operations via lio_listio, which has never been documented, requires the use of deliberately undefined constants (LIO_SYNC and LIO_MLOCK), and is arguably a bug. Reviewed by: jhb Differential Revision: https://reviews.freebsd.org/D27942 --- sys/kern/sys_socket.c | 8 +++----- sys/kern/vfs_aio.c | 48 ++++++++++++++---------------------------------- sys/sys/aio.h | 11 ++++++----- 3 files changed, 23 insertions(+), 44 deletions(-) diff --git a/sys/kern/sys_socket.c b/sys/kern/sys_socket.c index 18803b6a5ac0..e53b0367960b 100644 --- a/sys/kern/sys_socket.c +++ b/sys/kern/sys_socket.c @@ -775,10 +775,10 @@ soo_aio_cancel(struct kaiocb *job) so = job->fd_file->f_data; opcode = job->uaiocb.aio_lio_opcode; - if (opcode == LIO_READ || opcode == LIO_READV) + if (opcode & LIO_READ) sb = &so->so_rcv; else { - MPASS(opcode == LIO_WRITE || opcode == LIO_WRITEV); + MPASS(opcode & LIO_WRITE); sb = &so->so_snd; } @@ -808,13 +808,11 @@ soo_aio_queue(struct file *fp, struct kaiocb *job) if (error == 0) return (0); - switch (job->uaiocb.aio_lio_opcode) { + switch (job->uaiocb.aio_lio_opcode & (LIO_WRITE | LIO_READ)) { case LIO_READ: - case LIO_READV: sb = &so->so_rcv; break; case LIO_WRITE: - case LIO_WRITEV: sb = &so->so_snd; break; default: diff --git a/sys/kern/vfs_aio.c b/sys/kern/vfs_aio.c index bc0d7e04c9d5..9b45a06c5f9f 100644 --- a/sys/kern/vfs_aio.c +++ b/sys/kern/vfs_aio.c @@ -817,8 +817,7 @@ aio_process_rw(struct kaiocb *job) if (error != 0 && job->uiop->uio_resid != cnt) { if (error == ERESTART || error == EINTR || error == EWOULDBLOCK) error = 0; - if (error == EPIPE && - (opcode == LIO_WRITE || opcode == LIO_WRITEV)) { + if (error == EPIPE && (opcode & LIO_WRITE)) { PROC_LOCK(job->userproc); kern_psignal(job->userproc, SIGPIPE); PROC_UNLOCK(job->userproc); @@ -841,8 +840,7 @@ aio_process_sync(struct kaiocb *job) struct file *fp = job->fd_file; int error = 0; - KASSERT(job->uaiocb.aio_lio_opcode == LIO_SYNC || - job->uaiocb.aio_lio_opcode == LIO_DSYNC, + KASSERT(job->uaiocb.aio_lio_opcode & LIO_SYNC, ("%s: opcode %d", __func__, job->uaiocb.aio_lio_opcode)); td->td_ucred = job->cred; @@ -1239,8 +1237,7 @@ aio_qbio(struct proc *p, struct kaiocb *job) if (vp->v_bufobj.bo_bsize == 0) return (-1); - bio_cmd = opcode == LIO_WRITE || opcode == LIO_WRITEV ? BIO_WRITE : - BIO_READ; + bio_cmd = (opcode & LIO_WRITE) ? BIO_WRITE : BIO_READ; iovcnt = job->uiop->uio_iovcnt; if (iovcnt > max_buf_aio) return (-1); @@ -1422,7 +1419,7 @@ aiocb_copyin(struct aiocb *ujob, struct kaiocb *kjob, int type) error = copyin(ujob, kcb, sizeof(struct aiocb)); if (error) return (error); - if (type == LIO_READV || type == LIO_WRITEV) { + if (type & LIO_VECTORED) { /* malloc a uio and copy in the iovec */ error = copyinuio(__DEVOLATILE(struct iovec*, kcb->aio_iov), kcb->aio_iovcnt, &kjob->uiop); @@ -1609,7 +1606,7 @@ aio_aqueue(struct thread *td, struct aiocb *ujob, struct aioliojob *lj, if (error) goto err3; - if ((opcode == LIO_SYNC || opcode == LIO_DSYNC) && fp->f_vnode == NULL) { + if ((opcode & LIO_SYNC) && fp->f_vnode == NULL) { error = EINVAL; goto err3; } @@ -1669,14 +1666,10 @@ no_kqueue: job->jobflags = KAIOCB_QUEUEING; job->lio = lj; - switch (opcode) { - case LIO_READV: - case LIO_WRITEV: + if (opcode & LIO_VECTORED) { /* Use the uio copied in by aio_copyin */ MPASS(job->uiop != &job->uio && job->uiop != NULL); - break; - case LIO_READ: - case LIO_WRITE: + } else { /* Setup the inline uio */ job->iov[0].iov_base = (void *)(uintptr_t)job->uaiocb.aio_buf; job->iov[0].iov_len = job->uaiocb.aio_nbytes; @@ -1684,18 +1677,13 @@ no_kqueue: job->uio.uio_iovcnt = 1; job->uio.uio_resid = job->uaiocb.aio_nbytes; job->uio.uio_segflg = UIO_USERSPACE; - /* FALLTHROUGH */ - default: job->uiop = &job->uio; - break; } - switch (opcode) { + switch (opcode & (LIO_READ | LIO_WRITE)) { case LIO_READ: - case LIO_READV: job->uiop->uio_rw = UIO_READ; break; case LIO_WRITE: - case LIO_WRITEV: job->uiop->uio_rw = UIO_WRITE; break; } @@ -1813,21 +1801,14 @@ aio_queue_file(struct file *fp, struct kaiocb *job) return (EOPNOTSUPP); } - switch (job->uaiocb.aio_lio_opcode) { - case LIO_READ: - case LIO_READV: - case LIO_WRITE: - case LIO_WRITEV: + if (job->uaiocb.aio_lio_opcode & (LIO_WRITE | LIO_READ)) { aio_schedule(job, aio_process_rw); error = 0; - break; - case LIO_SYNC: - case LIO_DSYNC: + } else if (job->uaiocb.aio_lio_opcode & LIO_SYNC) { AIO_LOCK(ki); TAILQ_FOREACH(job2, &ki->kaio_jobqueue, plist) { if (job2->fd_file == job->fd_file && - job2->uaiocb.aio_lio_opcode != LIO_SYNC && - job2->uaiocb.aio_lio_opcode != LIO_DSYNC && + ((job2->uaiocb.aio_lio_opcode & LIO_SYNC) == 0) && job2->seqno < job->seqno) { job2->jobflags |= KAIOCB_CHECKSYNC; job->pending++; @@ -1847,8 +1828,7 @@ aio_queue_file(struct file *fp, struct kaiocb *job) AIO_UNLOCK(ki); aio_schedule(job, aio_process_sync); error = 0; - break; - default: + } else { error = EINVAL; } return (error); @@ -2498,7 +2478,7 @@ aio_biowakeup(struct bio *bp) */ if (flags & BIO_ERROR) atomic_set_int(&job->error, bio_error); - if (opcode == LIO_WRITE || opcode == LIO_WRITEV) + if (opcode & LIO_WRITE) atomic_add_int(&job->outblock, nblks); else atomic_add_int(&job->inblock, nblks); @@ -2836,7 +2816,7 @@ aiocb32_copyin(struct aiocb *ujob, struct kaiocb *kjob, int type) CP(job32, *kcb, aio_fildes); CP(job32, *kcb, aio_offset); CP(job32, *kcb, aio_lio_opcode); - if (type == LIO_READV || type == LIO_WRITEV) { + if (type & LIO_VECTORED) { iov32 = PTRIN(job32.aio_iov); CP(job32, *kcb, aio_iovcnt); /* malloc a uio and copy in the iovec */ diff --git a/sys/sys/aio.h b/sys/sys/aio.h index ee928b8bf846..d3e03efa310e 100644 --- a/sys/sys/aio.h +++ b/sys/sys/aio.h @@ -44,11 +44,12 @@ #define LIO_WRITE 0x1 #define LIO_READ 0x2 #if defined(_KERNEL) || defined(_WANT_ALL_LIO_OPCODES) -#define LIO_SYNC 0x3 -#define LIO_MLOCK 0x4 -#define LIO_WRITEV 0x5 -#define LIO_READV 0x6 -#define LIO_DSYNC 0x7 +#define LIO_VECTORED 0x4 +#define LIO_WRITEV (LIO_WRITE | LIO_VECTORED) +#define LIO_READV (LIO_READ | LIO_VECTORED) +#define LIO_SYNC 0x8 +#define LIO_DSYNC (0x10 | LIO_SYNC) +#define LIO_MLOCK 0x20 #endif /*