From owner-svn-src-all@FreeBSD.ORG Tue Feb 4 02:45:09 2014 Return-Path: Delivered-To: svn-src-all@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) (using TLSv1 with cipher ADH-AES256-SHA (256/256 bits)) (No client certificate requested) by hub.freebsd.org (Postfix) with ESMTPS id C1E5587D; Tue, 4 Feb 2014 02:45:09 +0000 (UTC) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:1900:2254:2068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by mx1.freebsd.org (Postfix) with ESMTPS id AE03B138B; Tue, 4 Feb 2014 02:45:09 +0000 (UTC) Received: from svn.freebsd.org ([127.0.1.70]) by svn.freebsd.org (8.14.8/8.14.8) with ESMTP id s142j99v018958; Tue, 4 Feb 2014 02:45:09 GMT (envelope-from neel@svn.freebsd.org) Received: (from neel@localhost) by svn.freebsd.org (8.14.8/8.14.8/Submit) id s142j865018954; Tue, 4 Feb 2014 02:45:08 GMT (envelope-from neel@svn.freebsd.org) Message-Id: <201402040245.s142j865018954@svn.freebsd.org> From: Neel Natu Date: Tue, 4 Feb 2014 02:45:08 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r261453 - head/sys/amd64/vmm/intel X-SVN-Group: head MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-all@freebsd.org X-Mailman-Version: 2.1.17 Precedence: list List-Id: "SVN commit messages for the entire src tree \(except for " user" and " projects" \)" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Tue, 04 Feb 2014 02:45:09 -0000 Author: neel Date: Tue Feb 4 02:45:08 2014 New Revision: 261453 URL: http://svnweb.freebsd.org/changeset/base/261453 Log: Avoid doing unnecessary nested TLB invalidations. Prior to this change the cached value of 'pm_eptgen' was tracked per-vcpu and per-hostcpu. In the degenerate case where 'N' vcpus were sharing a single hostcpu this could result in 'N - 1' unnecessary TLB invalidations. Since an 'invept' invalidates mappings for all VPIDs the first 'invept' is sufficient. Fix this by moving the 'eptgen[MAXCPU]' array from 'vmxctx' to 'struct vmx'. If it is known that an 'invept' is going to be done before entering the guest then it is safe to skip the 'invvpid'. The stat VPU_INVVPID_SAVED counts the number of 'invvpid' invalidations that were avoided because they were subsumed by an 'invept'. Discussed with: grehan Modified: head/sys/amd64/vmm/intel/vmx.c head/sys/amd64/vmm/intel/vmx.h head/sys/amd64/vmm/intel/vmx_genassym.c head/sys/amd64/vmm/intel/vmx_support.S Modified: head/sys/amd64/vmm/intel/vmx.c ============================================================================== --- head/sys/amd64/vmm/intel/vmx.c Tue Feb 4 02:41:54 2014 (r261452) +++ head/sys/amd64/vmm/intel/vmx.c Tue Feb 4 02:45:08 2014 (r261453) @@ -907,7 +907,6 @@ vmx_vminit(struct vm *vm, pmap_t pmap) panic("vmx_setup_cr4_shadow %d", error); vmx->ctx[i].pmap = pmap; - vmx->ctx[i].eptp = vmx->eptp; } return (vmx); @@ -955,20 +954,20 @@ vmx_astpending_trace(struct vmx *vmx, in #endif } +static VMM_STAT_INTEL(VCPU_INVVPID_SAVED, "Number of vpid invalidations saved"); + static void -vmx_set_pcpu_defaults(struct vmx *vmx, int vcpu) +vmx_set_pcpu_defaults(struct vmx *vmx, int vcpu, pmap_t pmap) { - int lastcpu; struct vmxstate *vmxstate; - struct invvpid_desc invvpid_desc = { 0 }; + struct invvpid_desc invvpid_desc; vmxstate = &vmx->state[vcpu]; - lastcpu = vmxstate->lastcpu; - vmxstate->lastcpu = curcpu; - - if (lastcpu == curcpu) + if (vmxstate->lastcpu == curcpu) return; + vmxstate->lastcpu = curcpu; + vmm_stat_incr(vmx->vm, vcpu, VCPU_MIGRATIONS, 1); vmcs_write(VMCS_HOST_TR_BASE, vmm_get_host_trbase()); @@ -991,8 +990,20 @@ vmx_set_pcpu_defaults(struct vmx *vmx, i * for "all" EP4TAs. */ if (vmxstate->vpid != 0) { - invvpid_desc.vpid = vmxstate->vpid; - invvpid(INVVPID_TYPE_SINGLE_CONTEXT, invvpid_desc); + if (pmap->pm_eptgen == vmx->eptgen[curcpu]) { + invvpid_desc._res1 = 0; + invvpid_desc._res2 = 0; + invvpid_desc.vpid = vmxstate->vpid; + invvpid(INVVPID_TYPE_SINGLE_CONTEXT, invvpid_desc); + } else { + /* + * The invvpid can be skipped if an invept is going to + * be performed before entering the guest. The invept + * will invalidate combined mappings tagged with + * 'vmx->eptp' for all vpids. + */ + vmm_stat_incr(vmx->vm, vcpu, VCPU_INVVPID_SAVED, 1); + } } } @@ -1859,8 +1870,6 @@ vmx_run(void *arg, int vcpu, register_t KASSERT(vmxctx->pmap == pmap, ("pmap %p different than ctx pmap %p", pmap, vmxctx->pmap)); - KASSERT(vmxctx->eptp == vmx->eptp, - ("eptp %p different than ctx eptp %#lx", eptp, vmxctx->eptp)); VMPTRLD(vmcs); @@ -1875,7 +1884,7 @@ vmx_run(void *arg, int vcpu, register_t vmcs_write(VMCS_HOST_CR3, rcr3()); vmcs_write(VMCS_GUEST_RIP, startrip); - vmx_set_pcpu_defaults(vmx, vcpu); + vmx_set_pcpu_defaults(vmx, vcpu, pmap); do { /* * Interrupts are disabled from this point on until the @@ -1910,7 +1919,7 @@ vmx_run(void *arg, int vcpu, register_t vmx_inject_interrupts(vmx, vcpu, vlapic); vmx_run_trace(vmx, vcpu); - rc = vmx_enter_guest(vmxctx, launched); + rc = vmx_enter_guest(vmxctx, vmx, launched); enable_intr(); Modified: head/sys/amd64/vmm/intel/vmx.h ============================================================================== --- head/sys/amd64/vmm/intel/vmx.h Tue Feb 4 02:41:54 2014 (r261452) +++ head/sys/amd64/vmm/intel/vmx.h Tue Feb 4 02:45:08 2014 (r261453) @@ -64,16 +64,13 @@ struct vmxctx { /* * XXX todo debug registers and fpu state */ - - int inst_fail_status; - long eptgen[MAXCPU]; /* cached pmap->pm_eptgen */ + int inst_fail_status; /* - * The 'eptp' and the 'pmap' do not change during the lifetime of - * the VM so it is safe to keep a copy in each vcpu's vmxctx. + * The pmap needs to be deactivated in vmx_exit_guest() + * so keep a copy of the 'pmap' in each vmxctx. */ - vm_paddr_t eptp; struct pmap *pmap; }; @@ -113,6 +110,7 @@ struct vmx { struct vmxstate state[VM_MAXCPU]; uint64_t eptp; struct vm *vm; + long eptgen[MAXCPU]; /* cached pmap->pm_eptgen */ }; CTASSERT((offsetof(struct vmx, vmcs) & PAGE_MASK) == 0); CTASSERT((offsetof(struct vmx, msr_bitmap) & PAGE_MASK) == 0); @@ -123,7 +121,7 @@ CTASSERT((offsetof(struct vmx, pir_desc[ #define VMX_VMRESUME_ERROR 1 #define VMX_VMLAUNCH_ERROR 2 #define VMX_INVEPT_ERROR 3 -int vmx_enter_guest(struct vmxctx *ctx, int launched); +int vmx_enter_guest(struct vmxctx *ctx, struct vmx *vmx, int launched); void vmx_exit_guest(void); void vmx_call_isr(uintptr_t entry); Modified: head/sys/amd64/vmm/intel/vmx_genassym.c ============================================================================== --- head/sys/amd64/vmm/intel/vmx_genassym.c Tue Feb 4 02:41:54 2014 (r261452) +++ head/sys/amd64/vmm/intel/vmx_genassym.c Tue Feb 4 02:45:08 2014 (r261453) @@ -68,10 +68,10 @@ ASSYM(VMXCTX_HOST_RBX, offsetof(struct v ASSYM(VMXCTX_HOST_RIP, offsetof(struct vmxctx, host_rip)); ASSYM(VMXCTX_INST_FAIL_STATUS, offsetof(struct vmxctx, inst_fail_status)); -ASSYM(VMXCTX_EPTGEN, offsetof(struct vmxctx, eptgen)); - ASSYM(VMXCTX_PMAP, offsetof(struct vmxctx, pmap)); -ASSYM(VMXCTX_EPTP, offsetof(struct vmxctx, eptp)); + +ASSYM(VMX_EPTGEN, offsetof(struct vmx, eptgen)); +ASSYM(VMX_EPTP, offsetof(struct vmx, eptp)); ASSYM(VM_FAIL_INVALID, VM_FAIL_INVALID); ASSYM(VM_FAIL_VALID, VM_FAIL_VALID); Modified: head/sys/amd64/vmm/intel/vmx_support.S ============================================================================== --- head/sys/amd64/vmm/intel/vmx_support.S Tue Feb 4 02:41:54 2014 (r261452) +++ head/sys/amd64/vmm/intel/vmx_support.S Tue Feb 4 02:45:08 2014 (r261453) @@ -97,7 +97,8 @@ /* * vmx_enter_guest(struct vmxctx *vmxctx, int launched) * %rdi: pointer to the 'vmxctx' - * %esi: launch state of the VMCS + * %rsi: pointer to the 'vmx' + * %edx: launch state of the VMCS * Interrupts must be disabled on entry. */ ENTRY(vmx_enter_guest) @@ -114,19 +115,19 @@ ENTRY(vmx_enter_guest) LK btsl %eax, PM_ACTIVE(%r11) /* - * If 'vmxctx->eptgen[curcpu]' is not identical to 'pmap->pm_eptgen' + * If 'vmx->eptgen[curcpu]' is not identical to 'pmap->pm_eptgen' * then we must invalidate all mappings associated with this EPTP. */ movq PM_EPTGEN(%r11), %r10 - cmpq %r10, VMXCTX_EPTGEN(%rdi, %rax, 8) + cmpq %r10, VMX_EPTGEN(%rsi, %rax, 8) je guest_restore - /* Refresh 'vmxctx->eptgen[curcpu]' */ - movq %r10, VMXCTX_EPTGEN(%rdi, %rax, 8) + /* Refresh 'vmx->eptgen[curcpu]' */ + movq %r10, VMX_EPTGEN(%rsi, %rax, 8) /* Setup the invept descriptor on the host stack */ mov %rsp, %r11 - movq VMXCTX_EPTP(%rdi), %rax + movq VMX_EPTP(%rsi), %rax movq %rax, -16(%r11) movq $0x0, -8(%r11) mov $0x1, %eax /* Single context invalidate */ @@ -134,7 +135,7 @@ ENTRY(vmx_enter_guest) jbe invept_error /* Check invept instruction error */ guest_restore: - cmpl $0, %esi + cmpl $0, %edx je do_launch VMX_GUEST_RESTORE