Date: Fri, 27 Sep 2013 10:06:14 GMT From: iori@FreeBSD.org To: svn-soc-all@FreeBSD.org Subject: socsvn commit: r257754 - in soc2013/iori/suspendresume/head: lib/libvmmapi sys/amd64/include sys/amd64/vmm sys/amd64/vmm/intel usr.sbin/bhyveload Message-ID: <201309271006.r8RA6EPx068864@socsvn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: iori Date: Fri Sep 27 10:06:14 2013 New Revision: 257754 URL: http://svnweb.FreeBSD.org/socsvn/?view=rev&rev=257754 Log: last commit in SoC. PR: Add register test api (not working). If it works well, I can confirm the register reloading is working correctly or not. What I have to check about register save/load are below: - The registers to be saved is enough or not - The registers to be loaded is enough or not - What is the best way to save the registers into a file And I make memory dump with dd command, but it's throttled, so I have to know the good practice to save it. (device state is included on vmm memory device). Modified: soc2013/iori/suspendresume/head/lib/libvmmapi/vmmapi.c soc2013/iori/suspendresume/head/sys/amd64/include/vmm.h soc2013/iori/suspendresume/head/sys/amd64/include/vmm_dev.h soc2013/iori/suspendresume/head/sys/amd64/vmm/intel/vmx.c soc2013/iori/suspendresume/head/sys/amd64/vmm/vmm.c soc2013/iori/suspendresume/head/sys/amd64/vmm/vmm_dev.c soc2013/iori/suspendresume/head/usr.sbin/bhyveload/Makefile soc2013/iori/suspendresume/head/usr.sbin/bhyveload/bhyveload.8 soc2013/iori/suspendresume/head/usr.sbin/bhyveload/bhyveload.c Modified: soc2013/iori/suspendresume/head/lib/libvmmapi/vmmapi.c ============================================================================== --- soc2013/iori/suspendresume/head/lib/libvmmapi/vmmapi.c Fri Sep 27 09:55:06 2013 (r257753) +++ soc2013/iori/suspendresume/head/lib/libvmmapi/vmmapi.c Fri Sep 27 10:06:14 2013 (r257754) @@ -124,6 +124,15 @@ } unsigned long +vm_set_allvstate(struct vmctx *ctx, struct vmstate *vmstate) +{ + int error; + + error = ioctl(ctx->fd, VM_RESTORE_VCPUSTATE, vmstate); + return error; +} + +unsigned long vm_get_allvstate(struct vmctx *ctx, struct vmstate *vmstate) { int error; @@ -132,6 +141,14 @@ return error; } +int +vm_polute_register(struct vmctx *ctx) +{ + int error; + error = ioctl(ctx->fd, VM_POLUTE_VCPUSTATE, NULL); + return error; +} + size_t vmm_get_mem_total(void) { @@ -316,6 +333,21 @@ } int +vm_set_register(struct vmctx *ctx, int vcpu, int reg, uint64_t val) +{ + int error; + struct vm_register vmreg; + + bzero(&vmreg, sizeof(vmreg)); + vmreg.cpuid = vcpu; + vmreg.regnum = reg; + vmreg.regval = val; + + error = ioctl(ctx->fd, VM_SET_REGISTER, &vmreg); + return (error); +} + +int vm_get_register(struct vmctx *ctx, int vcpu, int reg, uint64_t *ret_val) { int error; Modified: soc2013/iori/suspendresume/head/sys/amd64/include/vmm.h ============================================================================== --- soc2013/iori/suspendresume/head/sys/amd64/include/vmm.h Fri Sep 27 09:55:06 2013 (r257753) +++ soc2013/iori/suspendresume/head/sys/amd64/include/vmm.h Fri Sep 27 10:06:14 2013 (r257754) @@ -76,6 +76,7 @@ typedef int (*vmi_get_cap_t)(void *vmi, int vcpu, int num, int *retval); typedef int (*vmi_set_cap_t)(void *vmi, int vcpu, int num, int val); typedef int (*vmi_reg_copy_t)(void *vmi, int ncpu, struct vm_registers **vmcpu); +typedef int (*vmi_reg_restore_t)(void *vmi, int ncpu, struct vm_registers **vmcpu); struct vmm_ops { vmm_init_func_t init; /* module wide initialization */ @@ -93,6 +94,7 @@ vmi_get_cap_t vmgetcap; vmi_set_cap_t vmsetcap; vmi_reg_copy_t vreg_copy; + vmi_reg_restore_t vreg_restore; }; extern struct vmm_ops vmm_ops_intel; @@ -129,6 +131,7 @@ cpuset_t vm_active_cpus(struct vm *vm); struct vm_exit *vm_exitinfo(struct vm *vm, int vcpuid); int vm_save_vcpustate(struct vm *vm, int i, struct vm_vcpustate *vmstate); +int vm_restore_vcpustate(struct vm *vm, int i, struct vm_vcpustate *vmstate); /* * Return 1 if device indicated by bus/slot/func is supposed to be a Modified: soc2013/iori/suspendresume/head/sys/amd64/include/vmm_dev.h ============================================================================== --- soc2013/iori/suspendresume/head/sys/amd64/include/vmm_dev.h Fri Sep 27 09:55:06 2013 (r257753) +++ soc2013/iori/suspendresume/head/sys/amd64/include/vmm_dev.h Fri Sep 27 10:06:14 2013 (r257754) @@ -158,6 +158,7 @@ IOCNUM_SET_X2APIC_STATE, IOCNUM_GET_X2APIC_STATE, IOCNUM_SAVE_VCPUSTATE, + IOCNUM_RESTORE_VCPUSTATE, }; #define VM_RUN \ @@ -204,4 +205,6 @@ _IOWR('v', IOCNUM_GET_X2APIC_STATE, struct vm_x2apic) #define VM_SAVE_VCPUSTATE\ _IOWR('v', IOCNUM_SAVE_VCPUSTATE, struct vmstate) +#define VM_RESTORE_VCPUSTATE\ + _IOWR('v', IOCNUM_RESTORE_VCPUSTATE, struct vmstate) #endif Modified: soc2013/iori/suspendresume/head/sys/amd64/vmm/intel/vmx.c ============================================================================== --- soc2013/iori/suspendresume/head/sys/amd64/vmm/intel/vmx.c Fri Sep 27 09:55:06 2013 (r257753) +++ soc2013/iori/suspendresume/head/sys/amd64/vmm/intel/vmx.c Fri Sep 27 10:06:14 2013 (r257754) @@ -2007,8 +2007,8 @@ vmcpu[vcpu]->cr0 = (uint32_t)tmpreg; vmx_getreg(arg, vcpu, VM_REG_GUEST_CR3, &tmpreg); - vmcpu[vcpu]->cr3 = (uint32_t)tmpreg; + vmx_getreg(arg, vcpu, VM_REG_GUEST_CR4, &tmpreg); vmcpu[vcpu]->cr4 = (uint32_t)tmpreg; @@ -2034,6 +2034,68 @@ return 1; } +static int +vmx_cpu_regdisp(void *arg, int ncpu, struct vm_registers **vmcpu) +{ + struct vmx *vmx = arg; + int vcpu; + int buzy_count = 0; + int hostcpu; + + /*- This is a busy loop and must be fixed -*/ + for (vcpu = 0; vcpu < ncpu; vcpu++) + while (vcpu_is_running(vmx->vm, vcpu, &hostcpu)) + buzy_count++; + + printf("buzy_count waiting for vcpu stop is %d", buzy_count); + + /*- save all register and descriptor -*/ + for (vcpu = 0; vcpu < ncpu; vcpu++) { + vmx_setreg(arg, vcpu, VM_REG_GUEST_RAX, vmcpu[vcpu]->rax); + vmx_setreg(arg, vcpu, VM_REG_GUEST_RBX, vmcpu[vcpu]->rbx); + vmx_setreg(arg, vcpu, VM_REG_GUEST_RCX, vmcpu[vcpu]->rcx); + vmx_setreg(arg, vcpu, VM_REG_GUEST_RDX, vmcpu[vcpu]->rdx); + vmx_setreg(arg, vcpu, VM_REG_GUEST_RSI, vmcpu[vcpu]->rsi); + vmx_setreg(arg, vcpu, VM_REG_GUEST_RDI, vmcpu[vcpu]->rdi); + vmx_setreg(arg, vcpu, VM_REG_GUEST_RBP, vmcpu[vcpu]->rbp); + vmx_setreg(arg, vcpu, VM_REG_GUEST_RSP, vmcpu[vcpu]->rsp); + vmx_setreg(arg, vcpu, VM_REG_GUEST_R8, vmcpu[vcpu]->r8); + vmx_setreg(arg, vcpu, VM_REG_GUEST_R9, vmcpu[vcpu]->r9); + vmx_setreg(arg, vcpu, VM_REG_GUEST_R10, vmcpu[vcpu]->r10); + vmx_setreg(arg, vcpu, VM_REG_GUEST_R11, vmcpu[vcpu]->r11); + vmx_setreg(arg, vcpu, VM_REG_GUEST_R12, vmcpu[vcpu]->r12); + vmx_setreg(arg, vcpu, VM_REG_GUEST_R13, vmcpu[vcpu]->r13); + vmx_setreg(arg, vcpu, VM_REG_GUEST_R14, vmcpu[vcpu]->r14); + vmx_setreg(arg, vcpu, VM_REG_GUEST_R15, vmcpu[vcpu]->r15); + + vmx_setreg(arg, vcpu, VM_REG_GUEST_CR0, (uint64_t)vmcpu[vcpu]->cr0); + vmx_setreg(arg, vcpu, VM_REG_GUEST_CR3, (uint64_t)vmcpu[vcpu]->cr3); + vmx_setreg(arg, vcpu, VM_REG_GUEST_CR4, (uint64_t)vmcpu[vcpu]->cr4); + + vmx_setreg(arg, vcpu, VM_REG_GUEST_DR7, vmcpu[vcpu]->dr7); + vmx_setreg(arg, vcpu, VM_REG_GUEST_RIP, vmcpu[vcpu]->rip); + vmx_setreg(arg, vcpu, VM_REG_GUEST_RFLAGS, vmcpu[vcpu]->rflags); + vmx_setreg(arg, vcpu, VM_REG_GUEST_EFER, vmcpu[vcpu]->efer); + + /* + vmx_setseg(arg, vcpu, VM_REG_GUEST_ES, *(vmcpu[vcpu]->es)); + vmx_setseg(arg, vcpu, VM_REG_GUEST_CS, *(vmcpu[vcpu]->cs)); + vmx_setseg(arg, vcpu, VM_REG_GUEST_SS, *(vmcpu[vcpu]->ss)); + vmx_setseg(arg, vcpu, VM_REG_GUEST_DS, *(vmcpu[vcpu]->ds)); + vmx_setseg(arg, vcpu, VM_REG_GUEST_FS, *(vmcpu[vcpu]->fs)); + vmx_setseg(arg, vcpu, VM_REG_GUEST_GS, *(vmcpu[vcpu]->gs)); + + vmx_setseg(arg, vcpu, VM_REG_GUEST_TR, *(vmcpu[vcpu]->tr)); + */ + + vmx_setdesc(arg, vcpu, VM_REG_GUEST_LDTR, vmcpu[vcpu]->ldtr); + vmx_setdesc(arg, vcpu, VM_REG_GUEST_IDTR, vmcpu[vcpu]->idtr); + vmx_setdesc(arg, vcpu, VM_REG_GUEST_GDTR, vmcpu[vcpu]->gdtr); + } + + return 1; +} + struct vmm_ops vmm_ops_intel = { vmx_init, vmx_cleanup, @@ -2049,5 +2111,7 @@ vmx_inject, vmx_getcap, vmx_setcap, - vmx_cpu_regcopy + vmx_cpu_regcopy, + vmx_cpu_regdisp, }; + Modified: soc2013/iori/suspendresume/head/sys/amd64/vmm/vmm.c ============================================================================== --- soc2013/iori/suspendresume/head/sys/amd64/vmm/vmm.c Fri Sep 27 09:55:06 2013 (r257753) +++ soc2013/iori/suspendresume/head/sys/amd64/vmm/vmm.c Fri Sep 27 10:06:14 2013 (r257754) @@ -139,6 +139,8 @@ #define VREG_COPY(vmi, ncpu, vregs) \ (ops != NULL ? (*ops->vreg_copy)(vmi, ncpu, vregs) : ENXIO) +#define VREG_RESTORE(vmi, ncpu, vregs) \ + (ops != NULL ? (*ops->vreg_restore)(vmi, ncpu, vregs) : ENXIO) #define fpu_start_emulating() load_cr0(rcr0() | CR0_TS) #define fpu_stop_emulating() clts() @@ -1094,34 +1096,35 @@ struct vm_registers **vmregs; struct vm_vcpustate kvmcpu; struct vm_registers kvmregs; - struct vcpu *vcpu, *kvcpu; + struct vcpu *vcpu; + struct savefpu* kfpu; vm->control = REQ_SUSPEND; for (vcpuid = 0; vcpuid < ncpu; vcpuid++) vm_interrupt_hostcpu(vm, vcpuid); - vmregs = construct_vmcpu_snapshot(2); + vmregs = construct_vmcpu_snapshot(ncpu); + ret = VREG_COPY(vm->cookie, ncpu, vmregs); - printf("vmcpu is %p", vmcpu); - copyin(vmcpu, &kvmcpu, sizeof(struct vm_vcpuidstate)); - printf("vreg is %p", kvmcpu.regs); + /* we have to know the addresses of non-integer registers or structured + * register. The addresses are used to save these registers and after + * save operation, this addresses are written back into virtual registers + * struct. + */ + copyin(vmcpu, &kvmcpu, sizeof(struct vm_vcpustate)); for (vcpuid = 0; vcpuid < ncpu; vcpuid++) { vcpu = &vm->vcpu[vcpuid]; printf("start to copy in\n"); /* vm_segment_registers */ - printf("1 to copy in\n"); copyin(kvmcpu.regs, &kvmregs, - sizeof(struct vm_vcpuidstate)); - - printf("rax=%lu\n", kvmregs.rax); + sizeof(struct vm_vcpustate)); printf("start to copy out\n"); /* vm_segment_registers */ copyout(vmregs[vcpuid]->cs, cs = kvmregs.cs, sizeof(struct segment_register)); - printf("1 to copy out\n"); copyout(vmregs[vcpuid]->ds, ds = kvmregs.ds, sizeof(struct segment_register)); copyout(vmregs[vcpuid]->ss, ss = kvmregs.ss, @@ -1132,9 +1135,10 @@ sizeof(struct segment_register)); copyout(vmregs[vcpuid]->gs, gs = kvmregs.gs, sizeof(struct segment_register)); - copyout(vcpu->guestfpu, kvcpu = kvmregs.vfpu, - sizeof(struct segment_register*)); +/* vm registers */ + copyout(vcpu->guestfpu, kfpu = kvmregs.vfpu, + sizeof(struct segment_register*)); copyout(vmregs[vcpuid], vmcpu->regs, sizeof(struct vm_registers)); copyout(&cs, &(kvmregs.cs), sizeof(struct segment_register*)); @@ -1143,6 +1147,7 @@ copyout(&es, &(kvmregs.es), sizeof(struct segment_register*)); copyout(&fs, &(kvmregs.fs), sizeof(struct segment_register*)); copyout(&gs, &(kvmregs.gs), sizeof(struct segment_register*)); + copyout(&kfpu, &(kvmregs.vfpu), sizeof(struct savefpu*)); vmcpu = kvmcpu.nextcpu; @@ -1152,3 +1157,51 @@ return ret; }; +int +vm_restore_vcpustate(struct vm *vm, int ncpu, struct vm_vcpustate *vmcpu) +{ + int ret; + int vcpuid; + struct vm_registers **vmregs; + struct vm_vcpustate kvmcpu; + struct vm_registers kvmregs; + struct vcpu *vcpu; + + printf("restoring vm is not implemented yet.\n"); + vmregs = construct_vmcpu_snapshot(ncpu); + + copyin(vmcpu, &kvmcpu, sizeof(struct vm_vcpustate)); + + for (vcpuid = 0; vcpuid != ncpu; vcpuid++) { + vcpu = &vm->vcpu[vcpuid]; + copyin(kvmcpu.regs, &kvmregs, + sizeof(struct vm_vcpustate)); + +/* vm_segment_registers */ + copyin(kvmregs.cs, vmregs[vcpuid]->cs, + sizeof(struct segment_register)); + + copyin(kvmregs.ds, vmregs[vcpuid]->ds, + sizeof(struct segment_register)); + + copyin(kvmregs.ss, vmregs[vcpuid]->ss, + sizeof(struct segment_register)); + + copyin(kvmregs.es, vmregs[vcpuid]->es, + sizeof(struct segment_register)); + + copyin(kvmregs.ds, vmregs[vcpuid]->fs, + sizeof(struct segment_register)); + + copyin(kvmregs.ds, vmregs[vcpuid]->gs, + sizeof(struct segment_register)); + + VREG_RESTORE(vm->cookie, ncpu, vmregs); + + copyin(kvmregs.vfpu, vcpu->guestfpu, + sizeof(struct segment_register*)); + } + ret = 1; + return ret; +} + Modified: soc2013/iori/suspendresume/head/sys/amd64/vmm/vmm_dev.c ============================================================================== --- soc2013/iori/suspendresume/head/sys/amd64/vmm/vmm_dev.c Fri Sep 27 09:55:06 2013 (r257753) +++ soc2013/iori/suspendresume/head/sys/amd64/vmm/vmm_dev.c Fri Sep 27 10:06:14 2013 (r257754) @@ -351,6 +351,15 @@ error = vm_get_x2apic_state(sc->vm, x2apic->cpuid, &x2apic->state); break; + case VM_RESTORE_VCPUSTATE: + vmstate = (struct vmstate *)data; + if (vmstate->bhyve_version == 0) + error = vm_restore_vcpustate(sc->vm, vmstate->ncpu, vmstate->vmcpu); + else { + printf("unknown bhyve save file version!\n"); + error = -1; + } + break; case VM_SAVE_VCPUSTATE: vmstate = (struct vmstate *)data; vmstate->bhyve_version = 0; Modified: soc2013/iori/suspendresume/head/usr.sbin/bhyveload/Makefile ============================================================================== --- soc2013/iori/suspendresume/head/usr.sbin/bhyveload/Makefile Fri Sep 27 09:55:06 2013 (r257753) +++ soc2013/iori/suspendresume/head/usr.sbin/bhyveload/Makefile Fri Sep 27 10:06:14 2013 (r257754) @@ -1,7 +1,7 @@ # $FreeBSD$ PROG= bhyveload -SRCS= bhyveload.c +SRCS= bhyveload.c standalone.c MAN= bhyveload.8 DPADD+= ${LIBVMMAPI} Modified: soc2013/iori/suspendresume/head/usr.sbin/bhyveload/bhyveload.8 ============================================================================== --- soc2013/iori/suspendresume/head/usr.sbin/bhyveload/bhyveload.8 Fri Sep 27 09:55:06 2013 (r257753) +++ soc2013/iori/suspendresume/head/usr.sbin/bhyveload/bhyveload.8 Fri Sep 27 10:06:14 2013 (r257754) @@ -38,6 +38,7 @@ .Op Fl m Ar mem-size .Op Fl d Ar disk-path .Op Fl h Ar host-path +.Op Fl S Ar stand-binary:stand-addr .Ar vmname .Sh DESCRIPTION .Nm @@ -75,6 +76,13 @@ The .Ar host-path is the directory at the top of the guest's boot filesystem. +.It Fl S Ar stand-binary:stand-addr +The +.Ar stand-binary +is the pathname of the standalone guest program. +The +.Ar stand-addr +is the entrypoint address of the standalone guest program. .El .Sh EXAMPLES To create a virtual machine named Modified: soc2013/iori/suspendresume/head/usr.sbin/bhyveload/bhyveload.c ============================================================================== --- soc2013/iori/suspendresume/head/usr.sbin/bhyveload/bhyveload.c Fri Sep 27 09:55:06 2013 (r257753) +++ soc2013/iori/suspendresume/head/usr.sbin/bhyveload/bhyveload.c Fri Sep 27 10:06:14 2013 (r257754) @@ -95,6 +95,9 @@ static void cb_exit(void *arg, int v); +extern int stand_load(struct loader_callbacks *cb, char *image, + uint64_t addr); + /* * Console i/o callbacks */ @@ -552,6 +555,7 @@ fprintf(stderr, "usage: %s [-m mem-size][-d <disk-path>] [-h <host-path>] " + "[-S stand-binary:stand-addr] " "<vmname>\n", progname); exit(1); } @@ -563,6 +567,8 @@ void (*func)(struct loader_callbacks *, void *, int, int); uint64_t mem_size; int opt, error; + uint64_t stand_addr; + char *stand_image; char *disk_image; progname = argv[0]; @@ -570,7 +576,7 @@ mem_size = 256 * MB; disk_image = NULL; - while ((opt = getopt(argc, argv, "d:h:m:")) != -1) { + while ((opt = getopt(argc, argv, "d:h:m:S:")) != -1) { switch (opt) { case 'd': disk_image = optarg; @@ -583,7 +589,26 @@ case 'm': mem_size = strtoul(optarg, NULL, 0) * MB; break; - + + case 'S': { + char *addr_str; + + stand_image = strtok(optarg, ":"); + if (stand_image == NULL) { + usage(); + break; + } + + addr_str = strtok(NULL, ":"); + if (addr_str == NULL) { + usage(); + break; + } + stand_addr = strtoll(addr_str, NULL, 0); + + break; + } + case '?': usage(); } @@ -621,6 +646,11 @@ term.c_lflag &= ~(ICANON|ECHO); term.c_iflag &= ~ICRNL; tcsetattr(0, TCSAFLUSH, &term); + if (stand_image) { + if (stand_load(&cb, stand_image, stand_addr)) + exit(1); + exit(0); + } h = dlopen("/boot/userboot.so", RTLD_LOCAL); if (!h) { printf("%s\n", dlerror());
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201309271006.r8RA6EPx068864>