Date: Thu, 15 May 2014 08:56:07 -0700 From: Anish <akgupt3@gmail.com> To: Andriy Gapon <avg@freebsd.org> Cc: FreeBSD virtualization <freebsd-virtualization@freebsd.org>, Neel Natu <neel@freebsd.org> Subject: Re: bhyve: svm (amd-v) update Message-ID: <CALnRwMRpwc=DHib%2BeooftCkSP_K6XtVuR11AceDYju=mMBE2%2Bw@mail.gmail.com> In-Reply-To: <53748481.8010108@FreeBSD.org> References: <53748481.8010108@FreeBSD.org>
next in thread | previous in thread | raw e-mail | index | archive | help
[-- Attachment #1 --] Hi Andriy, Thanks for your interest in SVM port of bhyve. I do have patch to sync it to http://svnweb.freebsd.org/base?view=revision&revision=263780(3/26). If patches looks good to you, we can submit it. I have been testing it on Phenom box which lacks some of newer SVM features. Thanks and regards, Anish On Thu, May 15, 2014 at 2:10 AM, Andriy Gapon <avg@freebsd.org> wrote: > > It seems that the bhyve_svm branch is a bit behind the latest interface > changes > in head. Is anyone working on sync-ing up the branch with the head? > > Some examples: > - change of init method in vmm_ops > - addition of resume, vlapic_init and vlapic_cleanup methods to vmm_ops > - replacement of lapic_pending_intr and lapic_intr_accepted with > vlapic_pending_intr and vlapic_intr_accepted > - changes in struct vm_exit > - VMEXIT_EPT_FAULT ==> VMEXIT_NESTED_FAULT > > Thanks! > -- > Andriy Gapon > > _______________________________________________ > freebsd-virtualization@freebsd.org mailing list > http://lists.freebsd.org/mailman/listinfo/freebsd-virtualization > To unsubscribe, send any mail to " > freebsd-virtualization-unsubscribe@freebsd.org" > [-- Attachment #2 --] diff -aurP -x '*.orig' -x .svn /mnt/sync_next2/bhyve_svm_HEAD_r263780_base//sys/amd64/vmm/amd/npt.c /mnt/sync_next2/bhyve_svm_HEAD_r263780_WIP/sys/amd64/vmm/amd/npt.c --- /mnt/sync_next2/bhyve_svm_HEAD_r263780_base//sys/amd64/vmm/amd/npt.c 2014-04-11 13:51:45.000000000 +0000 +++ /mnt/sync_next2/bhyve_svm_HEAD_r263780_WIP/sys/amd64/vmm/amd/npt.c 2014-04-06 04:34:17.000000000 +0000 @@ -52,14 +52,17 @@ static int npt_flags; SYSCTL_INT(_hw_vmm_npt, OID_AUTO, pmap_flags, CTLFLAG_RD, &npt_flags, 0, NULL); + +#define NPT_IPIMASK 0xFF /* * AMD nested page table init. */ int -svm_npt_init(void) +svm_npt_init(int ipinum) { int enable_superpage = 1; + npt_flags = ipinum & NPT_IPIMASK; TUNABLE_INT_FETCH("hw.vmm.npt.enable_superpage", &enable_superpage); if (enable_superpage) npt_flags |= PMAP_PDE_SUPERPAGE; diff -aurP -x '*.orig' -x .svn /mnt/sync_next2/bhyve_svm_HEAD_r263780_base//sys/amd64/vmm/amd/npt.h /mnt/sync_next2/bhyve_svm_HEAD_r263780_WIP/sys/amd64/vmm/amd/npt.h --- /mnt/sync_next2/bhyve_svm_HEAD_r263780_base//sys/amd64/vmm/amd/npt.h 2014-04-11 13:51:45.000000000 +0000 +++ /mnt/sync_next2/bhyve_svm_HEAD_r263780_WIP/sys/amd64/vmm/amd/npt.h 2014-04-06 04:34:17.000000000 +0000 @@ -31,7 +31,7 @@ struct svm_softc; -int svm_npt_init(void); +int svm_npt_init(int ipinum); struct vmspace *svm_npt_alloc(vm_offset_t min, vm_offset_t max); void svm_npt_free(struct vmspace *vmspace); #endif /* _SVM_NPT_H_ */ diff -aurP -x '*.orig' -x .svn /mnt/sync_next2/bhyve_svm_HEAD_r263780_base//sys/amd64/vmm/amd/svm.c /mnt/sync_next2/bhyve_svm_HEAD_r263780_WIP/sys/amd64/vmm/amd/svm.c --- /mnt/sync_next2/bhyve_svm_HEAD_r263780_base//sys/amd64/vmm/amd/svm.c 2014-04-11 13:51:45.000000000 +0000 +++ /mnt/sync_next2/bhyve_svm_HEAD_r263780_WIP/sys/amd64/vmm/amd/svm.c 2014-04-12 11:51:53.000000000 +0000 @@ -46,6 +46,7 @@ #include <machine/specialreg.h> #include <machine/segments.h> #include <machine/vmm.h> +#include <machine/vmm_dev.h> #include <x86/apicreg.h> @@ -53,6 +54,8 @@ #include "vmm_msr.h" #include "vmm_stat.h" #include "vmm_ktr.h" +#include "vlapic.h" +#include "vlapic_priv.h" #include "x86.h" #include "vmcb.h" @@ -75,6 +78,7 @@ #define AMD_CPUID_SVM_PAUSE_FTH BIT(12) /* Pause filter threshold */ MALLOC_DEFINE(M_SVM, "svm", "svm"); +MALLOC_DEFINE(M_SVM_VLAPIC, "svm-vlapic", "svm-vlapic"); /* Per-CPU context area. */ extern struct pcpu __pcpu[]; @@ -249,7 +253,7 @@ * Enable SVM on CPU and initialize nested page table h/w. */ static int -svm_init(void) +svm_init(int ipinum) { int err; @@ -258,7 +262,7 @@ return (err); - svm_npt_init(); + svm_npt_init(ipinum); /* Start SVM on all CPUs */ smp_rendezvous(NULL, svm_enable, NULL, NULL); @@ -266,6 +270,11 @@ return (0); } +static void +svm_restore(void) +{ + svm_enable(NULL); +} /* * Get index and bit position for a MSR in MSR permission * bitmap. Two bits are used for each MSR, lower bit is @@ -365,7 +374,6 @@ return (0); } - /* * Initialise a virtual machine. */ @@ -519,6 +527,29 @@ } } +static enum vie_cpu_mode +svm_vcpu_mode(uint64_t efer) +{ + + if (efer & EFER_LMA) + return (CPU_MODE_64BIT); + else + return (CPU_MODE_COMPATIBILITY); +} + +static enum vie_paging_mode +svm_paging_mode(uint64_t cr0, uint64_t cr4, uint64_t efer) +{ + + if ((cr0 & CR0_PG) == 0) + return (PAGING_MODE_FLAT); + if ((cr4 & CR4_PAE) == 0) + return (PAGING_MODE_32); + if (efer & EFER_LME) + return (PAGING_MODE_64); + else + return (PAGING_MODE_PAE); +} /* * Determine the cause of virtual cpu exit and handle VMEXIT. * Return: false - Break vcpu execution loop and handle vmexit @@ -547,7 +578,7 @@ update_rip = true; loop = true; vmexit->exitcode = VM_EXITCODE_VMX; - vmexit->u.vmx.error = 0; + vmexit->u.vmx.status = 0; switch (code) { case VMCB_EXIT_MC: /* Machine Check. */ @@ -667,7 +698,6 @@ case VMCB_EXIT_NPF: loop = false; update_rip = false; - vmm_stat_incr(svm_sc->vm, vcpu, VMEXIT_EPT_FAULT, 1); if (info1 & VMCB_NPF_INFO1_RSV) { VCPU_CTR2(svm_sc->vm, vcpu, "SVM_ERR:NPT" @@ -686,15 +716,24 @@ vmexit->u.paging.gpa = info2; vmexit->u.paging.fault_type = svm_npf_paging(info1); + vmm_stat_incr(svm_sc->vm, vcpu, + VMEXIT_NESTED_FAULT, 1); } else if (svm_npf_emul_fault(info1)) { - VCPU_CTR3(svm_sc->vm, vcpu, "SVM:NPF-inst_emul," + VCPU_CTR3(svm_sc->vm, vcpu, "SVM:NPF inst_emul," "RIP:0x%lx INFO1:0x%lx INFO2:0x%lx .\n", state->rip, info1, info2); vmexit->exitcode = VM_EXITCODE_INST_EMUL; vmexit->u.inst_emul.gpa = info2; vmexit->u.inst_emul.gla = VIE_INVALID_GLA; vmexit->u.inst_emul.cr3 = state->cr3; + vmexit->u.inst_emul.cpu_mode = + svm_vcpu_mode(state->efer); + vmexit->u.inst_emul.paging_mode = + svm_paging_mode(state->cr0, state->cr4, + state->efer); vmexit->inst_length = VIE_INST_SIZE; + vmm_stat_incr(svm_sc->vm, vcpu, + VMEXIT_INST_EMUL, 1); } break; @@ -762,7 +801,7 @@ return (0); /* Inject NMI, vector number is not used.*/ - if (vmcb_eventinject(ctrl, VM_NMI, IDT_NMI, 0, false)) { + if (vmcb_eventinject(ctrl, VMCB_EVENTINJ_TYPE_NMI, IDT_NMI, 0, false)) { VCPU_CTR0(svm_sc->vm, vcpu, "SVM:NMI injection failed.\n"); return (EIO); } @@ -779,10 +818,11 @@ * Inject event to virtual cpu. */ static void -svm_inj_interrupts(struct svm_softc *svm_sc, int vcpu) +svm_inj_interrupts(struct svm_softc *svm_sc, int vcpu, struct vlapic *vlapic) { struct vmcb_ctrl *ctrl; struct vmcb_state *state; + struct vm_exception exc; int vector; KASSERT(vcpu < svm_sc->vcpu_cnt, ("Guest doesn't have VCPU%d", vcpu)); @@ -802,17 +842,26 @@ VCPU_CTR0(svm_sc->vm, vcpu, "SVM:Guest in interrupt shadow.\n"); return; } - + + if (vm_exception_pending(svm_sc->vm, vcpu, &exc)) { + KASSERT(exc.vector >= 0 && exc.vector < 32, + ("Exception vector% invalid", exc.vector)); + if (vmcb_eventinject(ctrl, VMCB_EVENTINJ_TYPE_EXCEPTION, + exc.vector, exc.error_code, + exc.error_code_valid)) { + VCPU_CTR1(svm_sc->vm, vcpu, "SVM:Exception%d injection" + " failed.\n", exc.vector); + return; + } + } /* NMI event has priority over interrupts.*/ if (svm_inject_nmi(svm_sc, vcpu)) { return; } - vector = lapic_pending_intr(svm_sc->vm, vcpu); - - /* No interrupt is pending. */ - if (vector < 0) - return; + /* Ask the local apic for a vector to inject */ + if (!vlapic_pending_intr(vlapic, &vector)) + return; if (vector < 32 || vector > 255) { VCPU_CTR1(svm_sc->vm, vcpu, "SVM_ERR:Event injection" @@ -826,14 +875,14 @@ return; } - if (vmcb_eventinject(ctrl, VM_HW_INTR, vector, 0, false)) { + if (vmcb_eventinject(ctrl, VMCB_EVENTINJ_TYPE_INTR, vector, 0, false)) { VCPU_CTR1(svm_sc->vm, vcpu, "SVM:Event injection failed to" " vector=%d.\n", vector); return; } /* Acknowledge that event is accepted.*/ - lapic_intr_accepted(svm_sc->vm, vcpu, vector); + vlapic_intr_accepted(vlapic, vector); VCPU_CTR1(svm_sc->vm, vcpu, "SVM:event injected,vector=%d.\n", vector); } @@ -887,7 +936,8 @@ * Start vcpu with specified RIP. */ static int -svm_vmrun(void *arg, int vcpu, register_t rip, pmap_t pmap) +svm_vmrun(void *arg, int vcpu, register_t rip, pmap_t pmap, + void *rend_cookie, void *suspended_cookie) { struct svm_regctx *hctx, *gctx; struct svm_softc *svm_sc; @@ -895,17 +945,21 @@ struct vmcb_state *state; struct vmcb_ctrl *ctrl; struct vm_exit *vmexit; + struct vlapic *vlapic; + struct vm *vm; uint64_t vmcb_pa; static uint64_t host_cr2; bool loop; /* Continue vcpu execution loop. */ loop = true; svm_sc = arg; - + vm = svm_sc->vm; + vcpustate = svm_get_vcpu(svm_sc, vcpu); state = svm_get_vmcb_state(svm_sc, vcpu); ctrl = svm_get_vmcb_ctrl(svm_sc, vcpu); - vmexit = vm_exitinfo(svm_sc->vm, vcpu); + vmexit = vm_exitinfo(vm, vcpu); + vlapic = vm_lapic(vm, vcpu); gctx = svm_get_guest_regctx(svm_sc, vcpu); hctx = &host_ctx[curcpu]; @@ -913,7 +967,7 @@ if (vcpustate->lastcpu != curcpu) { /* Virtual CPU is running on a diiferent CPU now.*/ - vmm_stat_incr(svm_sc->vm, vcpu, VCPU_MIGRATIONS, 1); + vmm_stat_incr(vm, vcpu, VCPU_MIGRATIONS, 1); /* * Flush all TLB mapping for this guest on this CPU, @@ -946,7 +1000,7 @@ } vcpustate->lastcpu = curcpu; - VCPU_CTR3(svm_sc->vm, vcpu, "SVM:Enter vmrun old RIP:0x%lx" + VCPU_CTR3(vm, vcpu, "SVM:Enter vmrun old RIP:0x%lx" " new RIP:0x%lx inst len=%d\n", state->rip, rip, vmexit->inst_length); /* Update Guest RIP */ @@ -957,9 +1011,25 @@ /* We are asked to give the cpu by scheduler. */ if (curthread->td_flags & (TDF_ASTPENDING | TDF_NEEDRESCHED)) { vmexit->exitcode = VM_EXITCODE_BOGUS; - vmm_stat_incr(svm_sc->vm, vcpu, VMEXIT_ASTPENDING, 1); - VCPU_CTR1(svm_sc->vm, vcpu, - "SVM: gave up CPU, RIP:0x%lx\n", state->rip); + vmm_stat_incr(vm, vcpu, VMEXIT_ASTPENDING, 1); + VCPU_CTR1(vm, vcpu, + "SVM: ASTPENDING, RIP:0x%lx\n", state->rip); + vmexit->rip = state->rip; + break; + } + + if (vcpu_suspended(suspended_cookie)) { + vmexit->exitcode = VM_EXITCODE_SUSPENDED; + vmexit->rip = state->rip; + break; + } + + if (vcpu_rendezvous_pending(rend_cookie)) { + vmexit->exitcode = VM_EXITCODE_RENDEZVOUS; + vmm_stat_incr(vm, vcpu, VMEXIT_RENDEZVOUS, 1); + VCPU_CTR1(vm, vcpu, + "SVM: VCPU rendezvous, RIP:0x%lx\n", + state->rip); vmexit->rip = state->rip; break; } @@ -968,7 +1038,7 @@ svm_handle_exitintinfo(svm_sc, vcpu); - (void)svm_inj_interrupts(svm_sc, vcpu); + (void)svm_inj_interrupts(svm_sc, vcpu, vlapic); /* Change TSS type to available.*/ setup_tss_type(); @@ -1017,7 +1087,7 @@ /* Handle #VMEXIT and if required return to user space. */ loop = svm_vmexit(svm_sc, vcpu, vmexit); vcpustate->loop++; - vmm_stat_incr(svm_sc->vm, vcpu, VMEXIT_COUNT, 1); + vmm_stat_incr(vm, vcpu, VMEXIT_COUNT, 1); } while (loop); @@ -1209,24 +1279,6 @@ } static int -svm_inject_event(void *arg, int vcpu, int type, int vector, - uint32_t error, int ec_valid) -{ - struct svm_softc *svm_sc; - struct vmcb_ctrl *ctrl; - - svm_sc = arg; - KASSERT(vcpu < svm_sc->vcpu_cnt, ("Guest doesn't have VCPU%d", vcpu)); - - ctrl = svm_get_vmcb_ctrl(svm_sc, vcpu); - VCPU_CTR3(svm_sc->vm, vcpu, "Injecting event type:0x%x vector:0x%x" - "error:0x%x\n", type, vector, error); - - return (vmcb_eventinject(ctrl, type, vector, error, - ec_valid ? TRUE : FALSE)); -} - -static int svm_setcap(void *arg, int vcpu, int type, int val) { struct svm_softc *svm_sc; @@ -1324,9 +1376,35 @@ return (0); } +static struct vlapic * +svm_vlapic_init(void *arg, int vcpuid) +{ + struct svm_softc *svm_sc; + struct vlapic *vlapic; + + svm_sc = arg; + vlapic = malloc(sizeof(struct vlapic), M_SVM_VLAPIC, M_WAITOK | M_ZERO); + vlapic->vm = svm_sc->vm; + vlapic->vcpuid = vcpuid; + vlapic->apic_page = (struct LAPIC *)&svm_sc->apic_page[vcpuid]; + + vlapic_init(vlapic); + + return (vlapic); +} + +static void +svm_vlapic_cleanup(void *arg, struct vlapic *vlapic) +{ + + vlapic_cleanup(vlapic); + free(vlapic, M_SVM_VLAPIC); +} + struct vmm_ops vmm_ops_amd = { svm_init, svm_cleanup, + svm_restore, svm_vminit, svm_vmrun, svm_vmcleanup, @@ -1334,9 +1412,10 @@ svm_setreg, svm_getdesc, svm_setdesc, - svm_inject_event, svm_getcap, svm_setcap, svm_npt_alloc, - svm_npt_free + svm_npt_free, + svm_vlapic_init, + svm_vlapic_cleanup }; diff -aurP -x '*.orig' -x .svn /mnt/sync_next2/bhyve_svm_HEAD_r263780_base//sys/amd64/vmm/amd/svm_softc.h /mnt/sync_next2/bhyve_svm_HEAD_r263780_WIP/sys/amd64/vmm/amd/svm_softc.h --- /mnt/sync_next2/bhyve_svm_HEAD_r263780_base//sys/amd64/vmm/amd/svm_softc.h 2014-04-11 13:51:45.000000000 +0000 +++ /mnt/sync_next2/bhyve_svm_HEAD_r263780_WIP/sys/amd64/vmm/amd/svm_softc.h 2014-04-06 04:34:17.000000000 +0000 @@ -61,6 +61,7 @@ */ uint8_t msr_bitmap[SVM_MSR_BITMAP_SIZE]; + uint8_t apic_page[VM_MAXCPU][PAGE_SIZE]; /* Nested Paging */ vm_offset_t nptp; diff -aurP -x '*.orig' -x .svn /mnt/sync_next2/bhyve_svm_HEAD_r263780_base//sys/amd64/vmm/amd/vmcb.c /mnt/sync_next2/bhyve_svm_HEAD_r263780_WIP/sys/amd64/vmm/amd/vmcb.c --- /mnt/sync_next2/bhyve_svm_HEAD_r263780_base//sys/amd64/vmm/amd/vmcb.c 2014-04-11 13:51:45.000000000 +0000 +++ /mnt/sync_next2/bhyve_svm_HEAD_r263780_WIP/sys/amd64/vmm/amd/vmcb.c 2014-04-06 04:34:17.000000000 +0000 @@ -354,25 +354,12 @@ * Inject an event to vcpu as described in section 15.20, "Event injection". */ int -vmcb_eventinject(struct vmcb_ctrl *ctrl, int type, int vector, +vmcb_eventinject(struct vmcb_ctrl *ctrl, int intr_type, int vector, uint32_t error, bool ec_valid) { - int intr_type; - - static uint8_t svm_intr_type_map[VM_EVENT_MAX] = { - -1, /* VM_EVENT_NONE */ - VMCB_EVENTINJ_TYPE_INTR, /* VM_HW_INTR */ - VMCB_EVENTINJ_TYPE_NMI, /* VM_NMI */ - VMCB_EVENTINJ_TYPE_EXCEPTION, /* VM_HW_EXCEPTION */ - VMCB_EVENTINJ_TYPE_INTn, /* VM_SW_INTR, INT */ - VMCB_EVENTINJ_TYPE_INTn, /* VM_PRIV_SW_EXCEPTION */ - VMCB_EVENTINJ_TYPE_INTn, /* VM_SW_EXCEPTION */ - }; - - intr_type = svm_intr_type_map[type]; if (intr_type < VMCB_EVENTINJ_TYPE_INTR || intr_type > VMCB_EVENTINJ_TYPE_INTn) { - ERR("Event:%d is not supported by SVM.\n", type); + ERR("Event:%d is not supported by SVM.\n", intr_type); return (EINVAL); }
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?CALnRwMRpwc=DHib%2BeooftCkSP_K6XtVuR11AceDYju=mMBE2%2Bw>