From owner-svn-src-head@freebsd.org Fri Sep 21 15:00:47 2018 Return-Path: Delivered-To: svn-src-head@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2610:1c1:1:606c::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id 5812A109C53F; Fri, 21 Sep 2018 15:00:47 +0000 (UTC) (envelope-from mjg@FreeBSD.org) Received: from mxrelay.nyi.freebsd.org (mxrelay.nyi.freebsd.org [IPv6:2610:1c1:1:606c::19:3]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client CN "mxrelay.nyi.freebsd.org", Issuer "Let's Encrypt Authority X3" (verified OK)) by mx1.freebsd.org (Postfix) with ESMTPS id 0EA0582445; Fri, 21 Sep 2018 15:00:47 +0000 (UTC) (envelope-from mjg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mxrelay.nyi.freebsd.org (Postfix) with ESMTPS id 097A26891; Fri, 21 Sep 2018 15:00:47 +0000 (UTC) (envelope-from mjg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id w8LF0kXN002712; Fri, 21 Sep 2018 15:00:46 GMT (envelope-from mjg@FreeBSD.org) Received: (from mjg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id w8LF0kdf002711; Fri, 21 Sep 2018 15:00:46 GMT (envelope-from mjg@FreeBSD.org) Message-Id: <201809211500.w8LF0kdf002711@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: mjg set sender to mjg@FreeBSD.org using -f From: Mateusz Guzik Date: Fri, 21 Sep 2018 15:00:46 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r338858 - head/sys/amd64/amd64 X-SVN-Group: head X-SVN-Commit-Author: mjg X-SVN-Commit-Paths: head/sys/amd64/amd64 X-SVN-Commit-Revision: 338858 X-SVN-Commit-Repository: base MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-head@freebsd.org X-Mailman-Version: 2.1.27 Precedence: list List-Id: SVN commit messages for the src tree for head/-current List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 21 Sep 2018 15:00:47 -0000 Author: mjg Date: Fri Sep 21 15:00:46 2018 New Revision: 338858 URL: https://svnweb.freebsd.org/changeset/base/338858 Log: amd64: even up copyin/copyout with memcpy + other cleanup - _fault handlers for both primitives are identical, provide just one - change the copying scheme to match memcpy (in particular jump avoidance for the most common case of multiply of 8) - stop re-reading pcb address on exit, just store it locally (in r9) Reviewed by: kib Approved by: re (gjb) Differential Revision: https://reviews.freebsd.org/D17265 Modified: head/sys/amd64/amd64/support.S Modified: head/sys/amd64/amd64/support.S ============================================================================== --- head/sys/amd64/amd64/support.S Fri Sep 21 13:44:05 2018 (r338857) +++ head/sys/amd64/amd64/support.S Fri Sep 21 15:00:46 2018 (r338858) @@ -309,9 +309,9 @@ END(fillw) */ .macro COPYOUT smap erms PUSH_FRAME_POINTER - movq PCPU(CURPCB),%rax + movq PCPU(CURPCB),%r9 /* Trap entry clears PSL.AC */ - movq $copyout_fault,PCB_ONFAULT(%rax) + movq $copy_fault,PCB_ONFAULT(%r9) testq %rdx,%rdx /* anything to do? */ jz 2f @@ -327,7 +327,7 @@ END(fillw) */ movq %rsi,%rax addq %rdx,%rax - jc copyout_fault + jc copy_fault /* * XXX STOP USING VM_MAXUSER_ADDRESS. * It is an end address, not a max, so every time it is used correctly it @@ -336,7 +336,7 @@ END(fillw) */ movq $VM_MAXUSER_ADDRESS,%rcx cmpq %rcx,%rax - ja copyout_fault + ja copy_fault xchgq %rdi,%rsi /* bcopy(%rsi, %rdi, %rdx) */ @@ -344,21 +344,27 @@ END(fillw) SMAP_DISABLE \smap .if \erms == 0 + cmpq $15,%rcx + jbe 1f shrq $3,%rcx rep movsq movb %dl,%cl andb $7,%cl - je 1f + jne 1f + SMAP_ENABLE \smap + xorl %eax,%eax + movq %rax,PCB_ONFAULT(%r9) + POP_FRAME_POINTER + ret .endif +1: rep movsb -1: SMAP_ENABLE \smap 2: xorl %eax,%eax - movq PCPU(CURPCB),%rdx - movq %rax,PCB_ONFAULT(%rdx) + movq %rax,PCB_ONFAULT(%r9) POP_FRAME_POINTER ret .endmacro @@ -379,22 +385,14 @@ ENTRY(copyout_smap_erms) COPYOUT smap=1 erms=1 END(copyout_smap_erms) - ALIGN_TEXT -copyout_fault: - movq PCPU(CURPCB),%rdx - movq $0,PCB_ONFAULT(%rdx) - movq $EFAULT,%rax - POP_FRAME_POINTER - ret - /* * copyin(from_user, to_kernel, len) * %rdi, %rsi, %rdx */ .macro COPYIN smap erms PUSH_FRAME_POINTER - movq PCPU(CURPCB),%rax - movq $copyin_fault,PCB_ONFAULT(%rax) + movq PCPU(CURPCB),%r9 + movq $copy_fault,PCB_ONFAULT(%r9) testq %rdx,%rdx /* anything to do? */ jz 2f @@ -403,10 +401,10 @@ copyout_fault: */ movq %rdi,%rax addq %rdx,%rax - jc copyin_fault + jc copy_fault movq $VM_MAXUSER_ADDRESS,%rcx cmpq %rcx,%rax - ja copyin_fault + ja copy_fault xchgq %rdi,%rsi movq %rdx,%rcx @@ -414,22 +412,28 @@ copyout_fault: SMAP_DISABLE \smap .if \erms == 0 + cmpq $15,%rcx + jbe 1f shrq $3,%rcx /* copy longword-wise */ rep movsq movb %al,%cl andb $7,%cl /* copy remaining bytes */ - je 1f + jne 1f + SMAP_ENABLE \smap + xorl %eax,%eax + movq %rax,PCB_ONFAULT(%r9) + POP_FRAME_POINTER + ret .endif +1: rep movsb -1: SMAP_ENABLE \smap 2: xorl %eax,%eax - movq PCPU(CURPCB),%rdx - movq %rax,PCB_ONFAULT(%rdx) + movq %rax,PCB_ONFAULT(%r9) POP_FRAME_POINTER ret .endmacro @@ -451,10 +455,10 @@ ENTRY(copyin_smap_erms) END(copyin_smap_erms) ALIGN_TEXT -copyin_fault: +copy_fault: movq PCPU(CURPCB),%rdx movq $0,PCB_ONFAULT(%rdx) - movq $EFAULT,%rax + movl $EFAULT,%eax POP_FRAME_POINTER ret