From owner-svn-src-head@FreeBSD.ORG Thu Jul 9 09:34:11 2009 Return-Path: Delivered-To: svn-src-head@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:4f8:fff6::34]) by hub.freebsd.org (Postfix) with ESMTP id BC7E2106566C; Thu, 9 Jul 2009 09:34:11 +0000 (UTC) (envelope-from kib@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:4f8:fff6::2c]) by mx1.freebsd.org (Postfix) with ESMTP id 9FBA98FC1A; Thu, 9 Jul 2009 09:34:11 +0000 (UTC) (envelope-from kib@FreeBSD.org) Received: from svn.freebsd.org (localhost [127.0.0.1]) by svn.freebsd.org (8.14.3/8.14.3) with ESMTP id n699YBVq078119; Thu, 9 Jul 2009 09:34:11 GMT (envelope-from kib@svn.freebsd.org) Received: (from kib@localhost) by svn.freebsd.org (8.14.3/8.14.3/Submit) id n699YBb4078107; Thu, 9 Jul 2009 09:34:11 GMT (envelope-from kib@svn.freebsd.org) Message-Id: <200907090934.n699YBb4078107@svn.freebsd.org> From: Konstantin Belousov Date: Thu, 9 Jul 2009 09:34:11 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org X-SVN-Group: head MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cc: Subject: svn commit: r195486 - in head/sys/amd64: amd64 ia32 include linux32 X-BeenThere: svn-src-head@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: SVN commit messages for the src tree for head/-current List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 09 Jul 2009 09:34:12 -0000 Author: kib Date: Thu Jul 9 09:34:11 2009 New Revision: 195486 URL: http://svn.freebsd.org/changeset/base/195486 Log: Restore the segment registers and segment base MSRs for amd64 syscall return path only when neither thread was context switched while executing syscall code nor syscall explicitely modified LDT or MSRs. Save segment registers in trap handlers before interrupts are enabled, to not allow context switches to happen before registers are saved. Use separated byte in pcb for indication of fast/full return, since pcb_flags are not synchronized with context switches. The change puts back syscall microbenchmark numbers that were slowed down after commit of the support for LDT on amd64. Reviewed by: jeff Tested (and tested, and tested ...) by: pho Approved by: re (kensmith) Modified: head/sys/amd64/amd64/cpu_switch.S head/sys/amd64/amd64/exception.S head/sys/amd64/amd64/genassym.c head/sys/amd64/amd64/machdep.c head/sys/amd64/amd64/sys_machdep.c head/sys/amd64/amd64/vm_machdep.c head/sys/amd64/ia32/ia32_exception.S head/sys/amd64/ia32/ia32_reg.c head/sys/amd64/ia32/ia32_signal.c head/sys/amd64/include/pcb.h head/sys/amd64/linux32/linux32_sysvec.c Modified: head/sys/amd64/amd64/cpu_switch.S ============================================================================== --- head/sys/amd64/amd64/cpu_switch.S Thu Jul 9 09:12:16 2009 (r195485) +++ head/sys/amd64/amd64/cpu_switch.S Thu Jul 9 09:34:11 2009 (r195486) @@ -97,6 +97,7 @@ END(cpu_throw) ENTRY(cpu_switch) /* Switch to new thread. First, save context. */ movq TD_PCB(%rdi),%r8 + movb $1,PCB_FULL_IRET(%r8) movq (%rsp),%rax /* Hardware registers */ movq %r15,PCB_R15(%r8) Modified: head/sys/amd64/amd64/exception.S ============================================================================== --- head/sys/amd64/amd64/exception.S Thu Jul 9 09:12:16 2009 (r195485) +++ head/sys/amd64/amd64/exception.S Thu Jul 9 09:34:11 2009 (r195486) @@ -162,19 +162,20 @@ IDTVEC(align) .globl alltraps .type alltraps,@function alltraps: + movq %rdi,TF_RDI(%rsp) testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ jz alltraps_testi /* already running with kernel GS.base */ swapgs + movq PCPU(CURPCB),%rdi + movb $0,PCB_FULL_IRET(%rdi) movw %fs,TF_FS(%rsp) movw %gs,TF_GS(%rsp) movw %es,TF_ES(%rsp) movw %ds,TF_DS(%rsp) alltraps_testi: testl $PSL_I,TF_RFLAGS(%rsp) - jz alltraps_pushregs + jz alltraps_pushregs_no_rdi sti -alltraps_pushregs: - movq %rdi,TF_RDI(%rsp) alltraps_pushregs_no_rdi: movq %rsi,TF_RSI(%rsp) movq %rdx,TF_RDX(%rsp) @@ -233,14 +234,17 @@ calltrap: .globl alltraps_noen .type alltraps_noen,@function alltraps_noen: + movq %rdi,TF_RDI(%rsp) testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ jz 1f /* already running with kernel GS.base */ swapgs + movq PCPU(CURPCB),%rdi + movb $0,PCB_FULL_IRET(%rdi) 1: movw %fs,TF_FS(%rsp) movw %gs,TF_GS(%rsp) movw %es,TF_ES(%rsp) movw %ds,TF_DS(%rsp) - jmp alltraps_pushregs + jmp alltraps_pushregs_no_rdi IDTVEC(dblfault) subq $TF_ERR,%rsp @@ -278,12 +282,13 @@ IDTVEC(dblfault) IDTVEC(page) subq $TF_ERR,%rsp movl $T_PAGEFLT,TF_TRAPNO(%rsp) + movq %rdi,TF_RDI(%rsp) /* free up a GP register */ testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ jz 1f /* already running with kernel GS.base */ swapgs -1: - movq %rdi,TF_RDI(%rsp) /* free up a GP register */ - movq %cr2,%rdi /* preserve %cr2 before .. */ + movq PCPU(CURPCB),%rdi + movb $0,PCB_FULL_IRET(%rdi) +1: movq %cr2,%rdi /* preserve %cr2 before .. */ movq %rdi,TF_ADDR(%rsp) /* enabling interrupts. */ movw %fs,TF_FS(%rsp) movw %gs,TF_GS(%rsp) @@ -311,7 +316,9 @@ IDTVEC(prot) testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ jz 2f /* already running with kernel GS.base */ 1: swapgs -2: movw %fs,TF_FS(%rsp) +2: movq PCPU(CURPCB),%rdi + movb $1,PCB_FULL_IRET(%rdi) /* always full iret from GPF */ + movw %fs,TF_FS(%rsp) movw %gs,TF_GS(%rsp) movw %es,TF_ES(%rsp) movw %ds,TF_DS(%rsp) @@ -341,6 +348,8 @@ IDTVEC(fast_syscall) movw %gs,TF_GS(%rsp) movw %es,TF_ES(%rsp) movw %ds,TF_DS(%rsp) + movq PCPU(CURPCB),%r11 + movb $0,PCB_FULL_IRET(%r11) sti movq $KUDSEL,TF_SS(%rsp) movq $KUCSEL,TF_CS(%rsp) @@ -644,7 +653,8 @@ doreti_exit: */ testb $SEL_RPL_MASK,TF_CS(%rsp) jz ld_regs - + cmpb $0,PCB_FULL_IRET(%r8) + je ld_regs testl $TF_HASSEGS,TF_FLAGS(%rsp) je set_segs Modified: head/sys/amd64/amd64/genassym.c ============================================================================== --- head/sys/amd64/amd64/genassym.c Thu Jul 9 09:12:16 2009 (r195485) +++ head/sys/amd64/amd64/genassym.c Thu Jul 9 09:34:11 2009 (r195486) @@ -141,6 +141,7 @@ ASSYM(PCB_DR3, offsetof(struct pcb, pcb_ ASSYM(PCB_DR6, offsetof(struct pcb, pcb_dr6)); ASSYM(PCB_DR7, offsetof(struct pcb, pcb_dr7)); ASSYM(PCB_TSSP, offsetof(struct pcb, pcb_tssp)); +ASSYM(PCB_FULL_IRET, offsetof(struct pcb, pcb_full_iret)); ASSYM(PCB_DBREGS, PCB_DBREGS); ASSYM(PCB_32BIT, PCB_32BIT); ASSYM(PCB_GS32BIT, PCB_GS32BIT); Modified: head/sys/amd64/amd64/machdep.c ============================================================================== --- head/sys/amd64/amd64/machdep.c Thu Jul 9 09:12:16 2009 (r195485) +++ head/sys/amd64/amd64/machdep.c Thu Jul 9 09:34:11 2009 (r195486) @@ -382,6 +382,7 @@ sendsig(sig_t catcher, ksiginfo_t *ksi, regs->tf_fs = _ufssel; regs->tf_gs = _ugssel; regs->tf_flags = TF_HASSEGS; + td->td_pcb->pcb_full_iret = 1; PROC_LOCK(p); mtx_lock(&psp->ps_mtx); } @@ -483,6 +484,7 @@ sigreturn(td, uap) signotify(td); PROC_UNLOCK(p); td->td_pcb->pcb_flags |= PCB_FULLCTX; + td->td_pcb->pcb_full_iret = 1; return (EJUSTRETURN); } @@ -853,6 +855,7 @@ exec_setregs(td, entry, stack, ps_string pcb->pcb_gsbase = 0; pcb->pcb_flags &= ~(PCB_32BIT | PCB_GS32BIT); pcb->pcb_initial_fpucw = __INITIAL_FPUCW__; + pcb->pcb_full_iret = 1; bzero((char *)regs, sizeof(struct trapframe)); regs->tf_rip = entry; @@ -2031,6 +2034,7 @@ set_mcontext(struct thread *td, const mc td->td_pcb->pcb_gsbase = mcp->mc_gsbase; } td->td_pcb->pcb_flags |= PCB_FULLCTX; + td->td_pcb->pcb_full_iret = 1; return (0); } Modified: head/sys/amd64/amd64/sys_machdep.c ============================================================================== --- head/sys/amd64/amd64/sys_machdep.c Thu Jul 9 09:12:16 2009 (r195485) +++ head/sys/amd64/amd64/sys_machdep.c Thu Jul 9 09:34:11 2009 (r195486) @@ -103,6 +103,7 @@ sysarch_ldt(struct thread *td, struct sy error = amd64_get_ldt(td, largs); break; case I386_SET_LDT: + td->td_pcb->pcb_full_iret = 1; if (largs->descs != NULL) { lp = (struct user_segment_descriptor *) kmem_alloc(kernel_map, largs->num * @@ -132,6 +133,7 @@ update_gdt_gsbase(struct thread *td, uin if (td != curthread) return; + td->td_pcb->pcb_full_iret = 1; critical_enter(); sd = PCPU_GET(gs32p); sd->sd_lobase = base & 0xffffff; @@ -146,6 +148,7 @@ update_gdt_fsbase(struct thread *td, uin if (td != curthread) return; + td->td_pcb->pcb_full_iret = 1; critical_enter(); sd = PCPU_GET(fs32p); sd->sd_lobase = base & 0xffffff; @@ -201,6 +204,7 @@ sysarch(td, uap) if (!error) { pcb->pcb_fsbase = i386base; td->td_frame->tf_fs = _ufssel; + pcb->pcb_full_iret = 1; update_gdt_fsbase(td, i386base); } break; @@ -212,6 +216,7 @@ sysarch(td, uap) error = copyin(uap->parms, &i386base, sizeof(i386base)); if (!error) { pcb->pcb_gsbase = i386base; + pcb->pcb_full_iret = 1; td->td_frame->tf_gs = _ugssel; update_gdt_gsbase(td, i386base); } @@ -225,6 +230,7 @@ sysarch(td, uap) if (!error) { if (a64base < VM_MAXUSER_ADDRESS) { pcb->pcb_fsbase = a64base; + pcb->pcb_full_iret = 1; td->td_frame->tf_fs = _ufssel; } else error = EINVAL; @@ -240,6 +246,7 @@ sysarch(td, uap) if (!error) { if (a64base < VM_MAXUSER_ADDRESS) { pcb->pcb_gsbase = a64base; + pcb->pcb_full_iret = 1; td->td_frame->tf_gs = _ugssel; } else error = EINVAL; @@ -525,6 +532,7 @@ amd64_set_ldt(td, uap, descs) uap->start, uap->num, (void *)uap->descs); #endif + td->td_pcb->pcb_full_iret = 1; p = td->td_proc; if (descs == NULL) { /* Free descriptors */ Modified: head/sys/amd64/amd64/vm_machdep.c ============================================================================== --- head/sys/amd64/amd64/vm_machdep.c Thu Jul 9 09:12:16 2009 (r195485) +++ head/sys/amd64/amd64/vm_machdep.c Thu Jul 9 09:34:11 2009 (r195486) @@ -186,6 +186,9 @@ cpu_fork(td1, p2, td2, flags) /* As an i386, do not copy io permission bitmap. */ pcb2->pcb_tssp = NULL; + /* New segment registers. */ + pcb2->pcb_full_iret = 1; + /* Copy the LDT, if necessary. */ mdp1 = &td1->td_proc->p_md; mdp2 = &p2->p_md; @@ -336,6 +339,7 @@ cpu_set_upcall(struct thread *td, struct */ bcopy(td0->td_pcb, pcb2, sizeof(*pcb2)); pcb2->pcb_flags &= ~PCB_FPUINITDONE; + pcb2->pcb_full_iret = 1; /* * Create a new fresh stack for the new thread. @@ -450,6 +454,7 @@ cpu_set_user_tls(struct thread *td, void } #endif td->td_pcb->pcb_fsbase = (register_t)tls_base; + td->td_pcb->pcb_full_iret = 1; return (0); } Modified: head/sys/amd64/ia32/ia32_exception.S ============================================================================== --- head/sys/amd64/ia32/ia32_exception.S Thu Jul 9 09:12:16 2009 (r195485) +++ head/sys/amd64/ia32/ia32_exception.S Thu Jul 9 09:34:11 2009 (r195486) @@ -42,10 +42,16 @@ SUPERALIGN_TEXT IDTVEC(int0x80_syscall) swapgs - sti pushq $2 /* sizeof "int 0x80" */ subq $TF_ERR,%rsp /* skip over tf_trapno */ movq %rdi,TF_RDI(%rsp) + movq PCPU(CURPCB),%rdi + movb $0,PCB_FULL_IRET(%rdi) + movw %fs,TF_FS(%rsp) + movw %gs,TF_GS(%rsp) + movw %es,TF_ES(%rsp) + movw %ds,TF_DS(%rsp) + sti movq %rsi,TF_RSI(%rsp) movq %rdx,TF_RDX(%rsp) movq %rcx,TF_RCX(%rsp) @@ -60,10 +66,6 @@ IDTVEC(int0x80_syscall) movq %r13,TF_R13(%rsp) movq %r14,TF_R14(%rsp) movq %r15,TF_R15(%rsp) - movw %fs,TF_FS(%rsp) - movw %gs,TF_GS(%rsp) - movw %es,TF_ES(%rsp) - movw %ds,TF_DS(%rsp) movl $TF_HASSEGS,TF_FLAGS(%rsp) FAKE_MCOUNT(TF_RIP(%rsp)) movq %rsp, %rdi Modified: head/sys/amd64/ia32/ia32_reg.c ============================================================================== --- head/sys/amd64/ia32/ia32_reg.c Thu Jul 9 09:12:16 2009 (r195485) +++ head/sys/amd64/ia32/ia32_reg.c Thu Jul 9 09:34:11 2009 (r195486) @@ -125,6 +125,7 @@ set_regs32(struct thread *td, struct reg tp->tf_fs = regs->r_fs; tp->tf_es = regs->r_es; tp->tf_ds = regs->r_ds; + td->td_pcb->pcb_full_iret = 1; tp->tf_flags = TF_HASSEGS; tp->tf_rdi = regs->r_edi; tp->tf_rsi = regs->r_esi; Modified: head/sys/amd64/ia32/ia32_signal.c ============================================================================== --- head/sys/amd64/ia32/ia32_signal.c Thu Jul 9 09:12:16 2009 (r195485) +++ head/sys/amd64/ia32/ia32_signal.c Thu Jul 9 09:34:11 2009 (r195486) @@ -159,6 +159,7 @@ ia32_get_mcontext(struct thread *td, str ia32_get_fpcontext(td, mcp); mcp->mc_fsbase = td->td_pcb->pcb_fsbase; mcp->mc_gsbase = td->td_pcb->pcb_gsbase; + td->td_pcb->pcb_full_iret = 1; return (0); } @@ -201,6 +202,7 @@ ia32_set_mcontext(struct thread *td, con tp->tf_rsp = mcp->mc_esp; tp->tf_ss = mcp->mc_ss; td->td_pcb->pcb_flags |= PCB_FULLCTX; + td->td_pcb->pcb_full_iret = 1; return (0); } @@ -394,6 +396,7 @@ freebsd4_ia32_sendsig(sig_t catcher, ksi regs->tf_ss = _udatasel; regs->tf_ds = _udatasel; regs->tf_es = _udatasel; + td->td_pcb->pcb_full_iret = 1; /* leave user %fs and %gs untouched */ PROC_LOCK(p); mtx_lock(&psp->ps_mtx); @@ -514,6 +517,7 @@ ia32_sendsig(sig_t catcher, ksiginfo_t * regs->tf_ss = _udatasel; regs->tf_ds = _udatasel; regs->tf_es = _udatasel; + td->td_pcb->pcb_full_iret = 1; /* XXXKIB leave user %fs and %gs untouched */ PROC_LOCK(p); mtx_lock(&psp->ps_mtx); @@ -611,6 +615,7 @@ freebsd4_freebsd32_sigreturn(td, uap) SIG_CANTMASK(td->td_sigmask); signotify(td); PROC_UNLOCK(p); + td->td_pcb->pcb_full_iret = 1; return (EJUSTRETURN); } #endif /* COMPAT_FREEBSD4 */ @@ -702,6 +707,7 @@ freebsd32_sigreturn(td, uap) SIG_CANTMASK(td->td_sigmask); signotify(td); PROC_UNLOCK(p); + td->td_pcb->pcb_full_iret = 1; return (EJUSTRETURN); } @@ -747,5 +753,6 @@ ia32_setregs(td, entry, stack, ps_string /* Return via doreti so that we can change to a different %cs */ pcb->pcb_flags |= PCB_FULLCTX | PCB_32BIT; pcb->pcb_flags &= ~PCB_GS32BIT; + td->td_pcb->pcb_full_iret = 1; td->td_retval[1] = 0; } Modified: head/sys/amd64/include/pcb.h ============================================================================== --- head/sys/amd64/include/pcb.h Thu Jul 9 09:12:16 2009 (r195485) +++ head/sys/amd64/include/pcb.h Thu Jul 9 09:34:11 2009 (r195486) @@ -72,12 +72,13 @@ struct pcb { struct savefpu pcb_save; uint16_t pcb_initial_fpucw; - caddr_t pcb_onfault; /* copyin/out fault recovery */ + caddr_t pcb_onfault; /* copyin/out fault recovery */ /* 32-bit segment descriptor */ struct user_segment_descriptor pcb_gs32sd; /* local tss, with i/o bitmap; NULL for common */ struct amd64tss *pcb_tssp; + char pcb_full_iret; }; struct xpcb { Modified: head/sys/amd64/linux32/linux32_sysvec.c ============================================================================== --- head/sys/amd64/linux32/linux32_sysvec.c Thu Jul 9 09:12:16 2009 (r195485) +++ head/sys/amd64/linux32/linux32_sysvec.c Thu Jul 9 09:34:11 2009 (r195486) @@ -423,6 +423,7 @@ linux_rt_sendsig(sig_t catcher, ksiginfo regs->tf_fs = _ufssel; regs->tf_gs = _ugssel; regs->tf_flags = TF_HASSEGS; + td->td_pcb->pcb_full_iret = 1; PROC_LOCK(p); mtx_lock(&psp->ps_mtx); } @@ -545,6 +546,7 @@ linux_sendsig(sig_t catcher, ksiginfo_t regs->tf_fs = _ufssel; regs->tf_gs = _ugssel; regs->tf_flags = TF_HASSEGS; + td->td_pcb->pcb_full_iret = 1; PROC_LOCK(p); mtx_lock(&psp->ps_mtx); } @@ -645,6 +647,7 @@ linux_sigreturn(struct thread *td, struc regs->tf_rflags = eflags; regs->tf_rsp = frame.sf_sc.sc_esp_at_signal; regs->tf_ss = frame.sf_sc.sc_ss; + td->td_pcb->pcb_full_iret = 1; return (EJUSTRETURN); } @@ -746,6 +749,7 @@ linux_rt_sigreturn(struct thread *td, st regs->tf_rflags = eflags; regs->tf_rsp = context->sc_esp_at_signal; regs->tf_ss = context->sc_ss; + td->td_pcb->pcb_full_iret = 1; /* * call sigaltstack & ignore results.. @@ -864,6 +868,7 @@ exec_linux_setregs(td, entry, stack, ps_ regs->tf_flags = TF_HASSEGS; regs->tf_cs = _ucode32sel; regs->tf_rbx = ps_strings; + td->td_pcb->pcb_full_iret = 1; load_cr0(rcr0() | CR0_MP | CR0_TS); fpstate_drop(td);