From owner-svn-src-user@FreeBSD.ORG Tue Dec 9 17:47:07 2008 Return-Path: Delivered-To: svn-src-user@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:4f8:fff6::34]) by hub.freebsd.org (Postfix) with ESMTP id 4CEF2106567D; Tue, 9 Dec 2008 17:47:06 +0000 (UTC) (envelope-from dfr@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:4f8:fff6::2c]) by mx1.freebsd.org (Postfix) with ESMTP id B329B8FC17; Tue, 9 Dec 2008 17:47:06 +0000 (UTC) (envelope-from dfr@FreeBSD.org) Received: from svn.freebsd.org (localhost [127.0.0.1]) by svn.freebsd.org (8.14.3/8.14.3) with ESMTP id mB9Hl6T6097539; Tue, 9 Dec 2008 17:47:06 GMT (envelope-from dfr@svn.freebsd.org) Received: (from dfr@localhost) by svn.freebsd.org (8.14.3/8.14.3/Submit) id mB9Hl6hU097525; Tue, 9 Dec 2008 17:47:06 GMT (envelope-from dfr@svn.freebsd.org) Message-Id: <200812091747.mB9Hl6hU097525@svn.freebsd.org> From: Doug Rabson Date: Tue, 9 Dec 2008 17:47:06 +0000 (UTC) To: src-committers@freebsd.org, svn-src-user@freebsd.org X-SVN-Group: user MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cc: Subject: svn commit: r185803 - in user/dfr/xenhvm/6/sys: conf dev/xen/blkfront dev/xen/console dev/xen/netfront dev/xen/xenpci i386/include/xen i386/xen xen xen/xenbus X-BeenThere: svn-src-user@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: "SVN commit messages for the experimental " user" src tree" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Tue, 09 Dec 2008 17:47:07 -0000 Author: dfr Date: Tue Dec 9 17:47:05 2008 New Revision: 185803 URL: http://svn.freebsd.org/changeset/base/185803 Log: Add support for suspend and resume. This currently only works for 64bit HV mode kernels. A 32bit PV mode kernel will suspend but fails to restore because Xen gets confused by the magic recursive mappings in the virtual address space. Added: user/dfr/xenhvm/6/sys/dev/xen/xenpci/machine_reboot.c Modified: user/dfr/xenhvm/6/sys/conf/files user/dfr/xenhvm/6/sys/dev/xen/blkfront/blkfront.c user/dfr/xenhvm/6/sys/dev/xen/console/xencons_ring.c user/dfr/xenhvm/6/sys/dev/xen/netfront/netfront.c user/dfr/xenhvm/6/sys/dev/xen/xenpci/evtchn.c user/dfr/xenhvm/6/sys/dev/xen/xenpci/xenpci.c user/dfr/xenhvm/6/sys/dev/xen/xenpci/xenpcivar.h user/dfr/xenhvm/6/sys/i386/include/xen/xenvar.h user/dfr/xenhvm/6/sys/i386/xen/clock.c user/dfr/xenhvm/6/sys/i386/xen/mp_machdep.c user/dfr/xenhvm/6/sys/i386/xen/xen_machdep.c user/dfr/xenhvm/6/sys/xen/reboot.c user/dfr/xenhvm/6/sys/xen/xenbus/xenbus_probe.c Modified: user/dfr/xenhvm/6/sys/conf/files ============================================================================== --- user/dfr/xenhvm/6/sys/conf/files Tue Dec 9 17:47:00 2008 (r185802) +++ user/dfr/xenhvm/6/sys/conf/files Tue Dec 9 17:47:05 2008 (r185803) @@ -2069,6 +2069,7 @@ dev/xen/netfront/netfront.c optional xen/gnttab.c optional xenhvm xen/features.c optional xenhvm dev/xen/xenpci/evtchn.c optional xenhvm +dev/xen/xenpci/machine_reboot.c optional xenhvm xen/evtchn/evtchn_dev.c optional xenhvm xen/reboot.c optional xenhvm xen/xenbus/xenbus_client.c optional xenhvm Modified: user/dfr/xenhvm/6/sys/dev/xen/blkfront/blkfront.c ============================================================================== --- user/dfr/xenhvm/6/sys/dev/xen/blkfront/blkfront.c Tue Dec 9 17:47:00 2008 (r185802) +++ user/dfr/xenhvm/6/sys/dev/xen/blkfront/blkfront.c Tue Dec 9 17:47:05 2008 (r185803) @@ -377,13 +377,12 @@ blkfront_resume(device_t dev) DPRINTK("blkfront_resume: %s\n", xenbus_get_node(dev)); - blkif_free(info, 1); - + blkif_free(info, info->connected == BLKIF_STATE_CONNECTED); err = talk_to_backend(dev, info); - if (!err) + if (info->connected == BLKIF_STATE_SUSPENDED && !err) blkif_recover(info); - return err; + return (err); } /* Common code used when first setting up, and when resuming. */ @@ -427,7 +426,7 @@ talk_to_backend(device_t dev, struct blk err = xenbus_transaction_end(xbt, 0); if (err) { - if (err == -EAGAIN) + if (err == EAGAIN) goto again; xenbus_dev_fatal(dev, err, "completing transaction"); goto destroy_blkring; Modified: user/dfr/xenhvm/6/sys/dev/xen/console/xencons_ring.c ============================================================================== --- user/dfr/xenhvm/6/sys/dev/xen/console/xencons_ring.c Tue Dec 9 17:47:00 2008 (r185802) +++ user/dfr/xenhvm/6/sys/dev/xen/console/xencons_ring.c Tue Dec 9 17:47:05 2008 (r185803) @@ -26,6 +26,7 @@ __FBSDID("$FreeBSD$"); #include #define console_evtchn console.domU.evtchn +static unsigned int console_irq; extern char *console_page; static inline struct xencons_interface * @@ -118,14 +119,17 @@ xencons_ring_init(void) err = bind_caller_port_to_irqhandler(xen_start_info->console_evtchn, "xencons", xencons_handle_input, NULL, - INTR_TYPE_MISC | INTR_MPSAFE, NULL); + INTR_TYPE_MISC | INTR_MPSAFE, &console_irq); if (err) { return err; } return 0; } -#ifdef notyet + +extern void xencons_suspend(void); +extern void xencons_resume(void); + void xencons_suspend(void) { @@ -133,7 +137,7 @@ xencons_suspend(void) if (!xen_start_info->console_evtchn) return; - unbind_evtchn_from_irqhandler(xen_start_info->console_evtchn, NULL); + unbind_from_irqhandler(console_irq); } void @@ -142,7 +146,7 @@ xencons_resume(void) (void)xencons_ring_init(); } -#endif + /* * Local variables: * mode: C Modified: user/dfr/xenhvm/6/sys/dev/xen/netfront/netfront.c ============================================================================== --- user/dfr/xenhvm/6/sys/dev/xen/netfront/netfront.c Tue Dec 9 17:47:00 2008 (r185802) +++ user/dfr/xenhvm/6/sys/dev/xen/netfront/netfront.c Tue Dec 9 17:47:05 2008 (r185803) @@ -430,13 +430,11 @@ netfront_attach(device_t dev) * leave the device-layer structures intact so that this is transparent to the * rest of the kernel. */ -static int +static int netfront_resume(device_t dev) { struct netfront_info *info = device_get_softc(dev); - - DPRINTK("%s\n", xenbus_get_node(dev)); - + netif_disconnect_backend(info); return (0); } @@ -576,9 +574,6 @@ setup_device(device_t dev, struct netfro if (error) goto fail; -#if 0 - network_connect(info); -#endif error = bind_listening_port_to_irqhandler(xenbus_get_otherend_id(dev), "xn", xn_intr, info, INTR_TYPE_NET | INTR_MPSAFE, &info->irq); @@ -1596,6 +1591,7 @@ network_connect(struct netfront_info *np /* Step 2: Rebuild the RX buffer freelist and the RX ring itself. */ for (requeue_idx = 0, i = 0; i < NET_RX_RING_SIZE; i++) { struct mbuf *m; + u_long pfn; if (np->rx_mbufs[i] == NULL) continue; @@ -1603,15 +1599,16 @@ network_connect(struct netfront_info *np m = np->rx_mbufs[requeue_idx] = xennet_get_rx_mbuf(np, i); ref = np->grant_rx_ref[requeue_idx] = xennet_get_rx_ref(np, i); req = RING_GET_REQUEST(&np->rx, requeue_idx); + pfn = vtophys(mtod(m, vm_offset_t)) >> PAGE_SHIFT; if (!np->copying_receiver) { gnttab_grant_foreign_transfer_ref(ref, xenbus_get_otherend_id(np->xbdev), - vtophys(mtod(m, vm_offset_t))); + pfn); } else { gnttab_grant_foreign_access_ref(ref, xenbus_get_otherend_id(np->xbdev), - vtophys(mtod(m, vm_offset_t)), 0); + PFNTOMFN(pfn), 0); } req->gref = ref; req->id = requeue_idx; @@ -1786,7 +1783,12 @@ static void netif_free(struct netfront_i static void netif_disconnect_backend(struct netfront_info *info) { - xn_stop(info); + XN_RX_LOCK(info); + XN_TX_LOCK(info); + netfront_carrier_off(info); + XN_TX_UNLOCK(info); + XN_RX_UNLOCK(info); + end_access(info->tx_ring_ref, info->tx.sring); end_access(info->rx_ring_ref, info->rx.sring); info->tx_ring_ref = GRANT_INVALID_REF; @@ -1794,12 +1796,8 @@ static void netif_disconnect_backend(str info->tx.sring = NULL; info->rx.sring = NULL; -#if 0 if (info->irq) unbind_from_irqhandler(info->irq); -#else - panic("FIX ME"); -#endif info->irq = 0; } Modified: user/dfr/xenhvm/6/sys/dev/xen/xenpci/evtchn.c ============================================================================== --- user/dfr/xenhvm/6/sys/dev/xen/xenpci/evtchn.c Tue Dec 9 17:47:00 2008 (r185802) +++ user/dfr/xenhvm/6/sys/dev/xen/xenpci/evtchn.c Tue Dec 9 17:47:05 2008 (r185803) @@ -75,6 +75,7 @@ static int evtchn_to_irq[NR_EVENT_CHANNE [0 ... NR_EVENT_CHANNELS-1] = -1 }; static struct mtx irq_alloc_lock; +static device_t xenpci_device; #define ARRAY_SIZE(a) (sizeof(a) / sizeof(a[0])) @@ -317,7 +318,7 @@ evtchn_interrupt(void *arg) handler = irq_evtchn[irq].handler; handler_arg = irq_evtchn[irq].arg; if (unlikely(handler == NULL)) { - printk("Xen IRQ%d (port %d) has no handler!\n", + printf("Xen IRQ%d (port %d) has no handler!\n", irq, port); mtx_unlock(&irq_evtchn[irq].lock); continue; @@ -345,8 +346,27 @@ evtchn_interrupt(void *arg) } } -void irq_resume(void) +void +irq_suspend(void) { + struct xenpci_softc *scp = device_get_softc(xenpci_device); + + /* + * Take our interrupt handler out of the list of handlers + * that can handle this irq. + */ + if (scp->intr_cookie != NULL) { + if (BUS_TEARDOWN_INTR(device_get_parent(xenpci_device), + xenpci_device, scp->res_irq, scp->intr_cookie) != 0) + printf("intr teardown failed.. continuing\n"); + scp->intr_cookie = NULL; + } +} + +void +irq_resume(void) +{ + struct xenpci_softc *scp = device_get_softc(xenpci_device); int evtchn, irq; for (evtchn = 0; evtchn < NR_EVENT_CHANNELS; evtchn++) { @@ -356,6 +376,10 @@ void irq_resume(void) for (irq = 0; irq < ARRAY_SIZE(irq_evtchn); irq++) irq_evtchn[irq].evtchn = 0; + + BUS_SETUP_INTR(device_get_parent(xenpci_device), + xenpci_device, scp->res_irq, INTR_TYPE_MISC, + evtchn_interrupt, NULL, &scp->intr_cookie); } int @@ -380,5 +404,7 @@ xenpci_irq_init(device_t device, struct if (error) return (error); + xenpci_device = device; + return (0); } Added: user/dfr/xenhvm/6/sys/dev/xen/xenpci/machine_reboot.c ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ user/dfr/xenhvm/6/sys/dev/xen/xenpci/machine_reboot.c Tue Dec 9 17:47:05 2008 (r185803) @@ -0,0 +1,125 @@ +#include +__FBSDID("$FreeBSD: user/dfr/xenhvm/6/sys/xen/evtchn/evtchn.c 184235 2008-10-25 00:25:25Z kmacy $"); + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include + +struct ap_suspend_info { + int do_spin; + atomic_t nr_spinning; +}; + +#ifdef CONFIG_SMP + +/* + * Spinning prevents, for example, APs touching grant table entries while + * the shared grant table is not mapped into the address space imemdiately + * after resume. + */ +static void ap_suspend(void *_info) +{ + struct ap_suspend_info *info = _info; + + BUG_ON(!irqs_disabled()); + + atomic_inc(&info->nr_spinning); + mb(); + + while (info->do_spin) + cpu_relax(); + + mb(); + atomic_dec(&info->nr_spinning); +} + +#define initiate_ap_suspend(i) smp_call_function(ap_suspend, i, 0, 0) + +#else /* !defined(CONFIG_SMP) */ + +#define initiate_ap_suspend(i) 0 + +#endif + +static int bp_suspend(void) +{ + int suspend_cancelled; + + suspend_cancelled = HYPERVISOR_suspend(0); + if (!suspend_cancelled) + xenpci_resume(); + + return suspend_cancelled; +} + +void +xen_suspend() +{ + int suspend_cancelled; + //struct ap_suspend_info info; + + if (DEVICE_SUSPEND(root_bus)) { + printf("xen_suspend: device_suspend failed\n"); + return; + } + + critical_enter(); + + /* Prevent any races with evtchn_interrupt() handler. */ + irq_suspend(); + +#if 0 + info.do_spin = 1; + atomic_set(&info.nr_spinning, 0); + smp_mb(); + + nr_cpus = num_online_cpus() - 1; + + err = initiate_ap_suspend(&info); + if (err < 0) { + critical_exit(); + //xenbus_suspend_cancel(); + return err; + } + + while (atomic_read(&info.nr_spinning) != nr_cpus) + cpu_relax(); +#endif + + disable_intr(); + suspend_cancelled = bp_suspend(); + //resume_notifier(suspend_cancelled); + enable_intr(); + +#if 0 + smp_mb(); + info.do_spin = 0; + while (atomic_read(&info.nr_spinning) != 0) + cpu_relax(); +#endif + + critical_exit(); + + if (!suspend_cancelled) + DEVICE_RESUME(root_bus); +#if 0 + else + xenbus_suspend_cancel(); +#endif +} Modified: user/dfr/xenhvm/6/sys/dev/xen/xenpci/xenpci.c ============================================================================== --- user/dfr/xenhvm/6/sys/dev/xen/xenpci/xenpci.c Tue Dec 9 17:47:00 2008 (r185802) +++ user/dfr/xenhvm/6/sys/dev/xen/xenpci/xenpci.c Tue Dec 9 17:47:05 2008 (r185803) @@ -44,6 +44,8 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include +#include #include #include @@ -63,6 +65,7 @@ __FBSDID("$FreeBSD$"); */ char *hypercall_stubs; shared_info_t *HYPERVISOR_shared_info; +static vm_paddr_t shared_info_pa; /* * The softc is automatically allocated by the parent bus using the @@ -75,7 +78,6 @@ static int xenpci_deallocate_resources(d static int xenpci_allocate_resources(device_t device); static int xenpci_attach(device_t device, struct xenpci_softc *scp); static int xenpci_detach(device_t device, struct xenpci_softc *scp); -static int xenpci_resume(device_t device, struct xenpci_softc *scp); static int xenpci_alloc_space_int(struct xenpci_softc *scp, size_t sz, u_long *pa); @@ -155,9 +157,8 @@ xenpci_pci_detach (device_t device) static int xenpci_pci_resume(device_t device) { - struct xenpci_softc *scp = DEVICE2SOFTC(device); - return (xenpci_resume(device, scp)); + return (bus_generic_resume(device)); } /* @@ -251,7 +252,6 @@ xenpci_attach(device_t device, struct xe { struct xen_add_to_physmap xatp; vm_offset_t shared_va; - vm_paddr_t shared_pa; if (xenpci_allocate_resources(device)) goto errexit; @@ -263,17 +263,17 @@ xenpci_attach(device_t device, struct xe setup_xen_features(); - xenpci_alloc_space_int(scp, PAGE_SIZE, &shared_pa); + xenpci_alloc_space_int(scp, PAGE_SIZE, &shared_info_pa); xatp.domid = DOMID_SELF; xatp.idx = 0; xatp.space = XENMAPSPACE_shared_info; - xatp.gpfn = shared_pa >> PAGE_SHIFT; + xatp.gpfn = shared_info_pa >> PAGE_SHIFT; if (HYPERVISOR_memory_op(XENMEM_add_to_physmap, &xatp)) panic("HYPERVISOR_memory_op failed"); shared_va = kmem_alloc_nofault(kernel_map, PAGE_SIZE); - pmap_kenter(shared_va, shared_pa); + pmap_kenter(shared_va, shared_info_pa); HYPERVISOR_shared_info = (void *) shared_va; /* @@ -316,13 +316,6 @@ xenpci_detach(device_t device, struct xe } static int -xenpci_resume(device_t device, struct xenpci_softc *scp) -{ - - xenpci_resume_hypercall_stubs(device, scp); -} - -static int xenpci_allocate_resources(device_t device) { int error; @@ -412,11 +405,26 @@ xenpci_alloc_space(size_t sz, vm_paddr_t } void -printk(const char *fmt, ...) +xenpci_resume() { - __va_list ap; + device_t device = devclass_get_device(xenpci_devclass, 0); + struct xenpci_softc *scp = DEVICE2SOFTC(device); + struct xen_add_to_physmap xatp; + + xenpci_resume_hypercall_stubs(device, scp); - va_start(ap, fmt); - vprintf(fmt, ap); - va_end(ap); + xatp.domid = DOMID_SELF; + xatp.idx = 0; + xatp.space = XENMAPSPACE_shared_info; + xatp.gpfn = shared_info_pa >> PAGE_SHIFT; + if (HYPERVISOR_memory_op(XENMEM_add_to_physmap, &xatp)) + panic("HYPERVISOR_memory_op failed"); + + pmap_kenter((vm_offset_t) HYPERVISOR_shared_info, shared_info_pa); + + xenpci_set_callback(device); + + gnttab_resume(); + irq_resume(); } + Modified: user/dfr/xenhvm/6/sys/dev/xen/xenpci/xenpcivar.h ============================================================================== --- user/dfr/xenhvm/6/sys/dev/xen/xenpci/xenpcivar.h Tue Dec 9 17:47:00 2008 (r185802) +++ user/dfr/xenhvm/6/sys/dev/xen/xenpci/xenpcivar.h Tue Dec 9 17:47:05 2008 (r185803) @@ -41,3 +41,5 @@ struct xenpci_softc { extern int xenpci_irq_init(device_t device, struct xenpci_softc *scp); extern int xenpci_alloc_space(size_t sz, vm_paddr_t *pa); +extern void xenpci_resume(void); +extern void xen_suspend(void); Modified: user/dfr/xenhvm/6/sys/i386/include/xen/xenvar.h ============================================================================== --- user/dfr/xenhvm/6/sys/i386/include/xen/xenvar.h Tue Dec 9 17:47:00 2008 (r185802) +++ user/dfr/xenhvm/6/sys/i386/include/xen/xenvar.h Tue Dec 9 17:47:05 2008 (r185803) @@ -39,6 +39,8 @@ extern int xendebug_flags; #endif extern xen_pfn_t *xen_phys_machine; +extern xen_pfn_t *xen_pfn_to_mfn_frame_list[16]; +extern xen_pfn_t *xen_pfn_to_mfn_frame_list_list; #if 0 #define TRACE_ENTER XENPRINTF("(file=%s, line=%d) entered %s\n", __FILE__, __LINE__, __FUNCTION__) Modified: user/dfr/xenhvm/6/sys/i386/xen/clock.c ============================================================================== --- user/dfr/xenhvm/6/sys/i386/xen/clock.c Tue Dec 9 17:47:00 2008 (r185802) +++ user/dfr/xenhvm/6/sys/i386/xen/clock.c Tue Dec 9 17:47:05 2008 (r185803) @@ -131,7 +131,7 @@ static u_char timer2_state = RELEASED; static struct timespec shadow_tv; static uint32_t shadow_tv_version; /* XXX: lazy locking */ static uint64_t processed_system_time; /* stime (ns) at last processing. */ - +static unsigned int time_irq; #ifdef XEN_PRIVILEGED_GUEST static struct mtx clock_lock; @@ -839,23 +839,25 @@ static struct vcpu_set_periodic_timer xe void cpu_initclocks(void) { - int time_irq; xen_set_periodic_tick.period_ns = NS_PER_TICK; HYPERVISOR_vcpu_op(VCPUOP_set_periodic_timer, 0, &xen_set_periodic_tick); - if ((time_irq = bind_virq_to_irqhandler(VIRQ_TIMER, 0, "clk", - (driver_intr_t *)clkintr, INTR_TYPE_CLK | INTR_FAST, - NULL)) < 0) { + if (time_irq) + unbind_from_irqhandler(time_irq); + time_irq = 0; + + if (bind_virq_to_irqhandler(VIRQ_TIMER, 0, "clk", + (driver_intr_t *)clkintr, INTR_TYPE_CLK | INTR_FAST, + &time_irq)) { panic("failed to register clock interrupt\n"); } /* should fast clock be enabled ? */ } - int ap_cpu_initclocks(int cpu) { Modified: user/dfr/xenhvm/6/sys/i386/xen/mp_machdep.c ============================================================================== --- user/dfr/xenhvm/6/sys/i386/xen/mp_machdep.c Tue Dec 9 17:47:00 2008 (r185802) +++ user/dfr/xenhvm/6/sys/i386/xen/mp_machdep.c Tue Dec 9 17:47:05 2008 (r185803) @@ -567,7 +567,7 @@ xen_smp_intr_init(unsigned int cpu) INTR_FAST|INTR_TYPE_TTY|INTR_MPSAFE, &pc->pc_resched_irq); printf("cpu=%d irq=%d vector=%d\n", - cpu, rc, RESCHEDULE_VECTOR); + cpu, pc->pc_resched_irq, RESCHEDULE_VECTOR); sprintf(callfunc_name[cpu], "callfunc%u", cpu); rc = bind_ipi_to_irqhandler(CALL_FUNCTION_VECTOR, cpu, @@ -578,7 +578,7 @@ xen_smp_intr_init(unsigned int cpu) goto fail; printf("cpu=%d irq=%d vector=%d\n", - cpu, rc, CALL_FUNCTION_VECTOR); + cpu, pc->pc_callfunc_irq, CALL_FUNCTION_VECTOR); if ((cpu != 0) && ((rc = ap_cpu_initclocks(cpu)) != 0)) goto fail; Modified: user/dfr/xenhvm/6/sys/i386/xen/xen_machdep.c ============================================================================== --- user/dfr/xenhvm/6/sys/i386/xen/xen_machdep.c Tue Dec 9 17:47:00 2008 (r185802) +++ user/dfr/xenhvm/6/sys/i386/xen/xen_machdep.c Tue Dec 9 17:47:05 2008 (r185803) @@ -88,6 +88,8 @@ start_info_t *xen_start_info; shared_info_t *HYPERVISOR_shared_info; xen_pfn_t *xen_machine_phys = machine_to_phys_mapping; xen_pfn_t *xen_phys_machine; +xen_pfn_t *xen_pfn_to_mfn_frame_list[16]; +xen_pfn_t *xen_pfn_to_mfn_frame_list_list; int preemptable, init_first; extern unsigned int avail_space; @@ -842,6 +844,39 @@ shift_phys_machine(unsigned long *phys_m } #endif /* ADD_ISA_HOLE */ +/* + * Build a directory of the pages that make up our Physical to Machine + * mapping table. The Xen suspend/restore code uses this to find our + * mapping table. + */ +static void +init_frame_list_list(void *arg) +{ + unsigned long nr_pages = xen_start_info->nr_pages; +#define FPP (PAGE_SIZE/sizeof(xen_pfn_t)) + int i, j, k; + + xen_pfn_to_mfn_frame_list_list = malloc(PAGE_SIZE, M_DEVBUF, M_WAITOK); + for (i = 0, j = 0, k = -1; i < nr_pages; + i += FPP, j++) { + if ((j & (FPP - 1)) == 0) { + k++; + xen_pfn_to_mfn_frame_list[k] = + malloc(PAGE_SIZE, M_DEVBUF, M_WAITOK); + xen_pfn_to_mfn_frame_list_list[k] = + VTOMFN(xen_pfn_to_mfn_frame_list[k]); + j = 0; + } + xen_pfn_to_mfn_frame_list[k][j] = + VTOMFN(&xen_phys_machine[i]); + } + + HYPERVISOR_shared_info->arch.max_pfn = nr_pages; + HYPERVISOR_shared_info->arch.pfn_to_mfn_frame_list_list + = VTOMFN(xen_pfn_to_mfn_frame_list_list); +} +SYSINIT(init_fll, SI_SUB_DEVFS, SI_ORDER_ANY, init_frame_list_list, NULL); + extern unsigned long physfree; int pdir, curoffset; @@ -975,7 +1010,7 @@ initvalues(start_info_t *startinfo) IdlePTDnew = (pd_entry_t *)cur_space; cur_space += 4*PAGE_SIZE; bzero(IdlePTDnew, 4*PAGE_SIZE); - for (i = 0; i < 4; i++) + for (i = 0; i < 4; i++) IdlePTDnewma[i] = xpmap_ptom(VTOP((uint8_t *)IdlePTDnew + i*PAGE_SIZE)); /* @@ -1020,8 +1055,6 @@ initvalues(start_info_t *startinfo) PT_UPDATES_FLUSH(); - - memcpy(((uint8_t *)IdlePTDnew) + ((unsigned int)(KERNBASE >> 18)), ((uint8_t *)IdlePTD) + ((KERNBASE >> 18) & PAGE_MASK), l1_pages*sizeof(pt_entry_t)); @@ -1082,14 +1115,12 @@ initvalues(start_info_t *startinfo) printk("#4\n"); - xen_store_ma = (((vm_paddr_t)xen_start_info->store_mfn) << PAGE_SHIFT); PT_SET_MA(xen_store, xen_store_ma | PG_KERNEL); console_page_ma = (((vm_paddr_t)xen_start_info->console.domU.mfn) << PAGE_SHIFT); PT_SET_MA(console_page, console_page_ma | PG_KERNEL); printk("#5\n"); - HYPERVISOR_shared_info->arch.pfn_to_mfn_frame_list_list = (unsigned long)xen_phys_machine; set_iopl.iopl = 1; err = HYPERVISOR_physdev_op(PHYSDEVOP_SET_IOPL, &set_iopl); Modified: user/dfr/xenhvm/6/sys/xen/reboot.c ============================================================================== --- user/dfr/xenhvm/6/sys/xen/reboot.c Tue Dec 9 17:47:00 2008 (r185802) +++ user/dfr/xenhvm/6/sys/xen/reboot.c Tue Dec 9 17:47:05 2008 (r185803) @@ -34,13 +34,34 @@ __FBSDID("$FreeBSD$"); #include -#include +#include #include #include +#include #include +#include +#include +#include +#include +#include +#include +#include #include +#include +#include + +#ifdef XENHVM + +#include + +#else + +static void xen_suspend(void); + +#endif + static void shutdown_handler(struct xenbus_watch *watch, const char **vec, unsigned int len) @@ -91,22 +112,35 @@ shutdown_handler(struct xenbus_watch *wa printf("Ignoring shutdown request: %s\n", str); goto done; } -#ifdef notyet - if (howto == -1) { - do_suspend(NULL); - goto done; - } -#else + if (howto == -1) { - printf("suspend not currently supported\n"); + xen_suspend(); goto done; } -#endif + shutdown_nice(howto); done: free(str, M_DEVBUF); } +#ifndef XENHVM + +/* + * In HV mode, we let acpi take care of halts and reboots. + */ + +static void +xen_shutdown_final(void *arg, int howto) +{ + + if (howto & (RB_HALT | RB_POWEROFF)) + HYPERVISOR_shutdown(SHUTDOWN_poweroff); + else + HYPERVISOR_shutdown(SHUTDOWN_reboot); +} + +#endif + static struct xenbus_watch shutdown_watch = { .node = "control/shutdown", .callback = shutdown_handler @@ -118,80 +152,61 @@ setup_shutdown_watcher(void *unused) if (register_xenbus_watch(&shutdown_watch)) printf("Failed to set shutdown watcher\n"); +#ifndef XENHVM + EVENTHANDLER_REGISTER(shutdown_final, xen_shutdown_final, NULL, + SHUTDOWN_PRI_LAST); +#endif } SYSINIT(shutdown, SI_SUB_PSEUDO, SI_ORDER_ANY, setup_shutdown_watcher, NULL); -#ifdef notyet +#ifndef XENHVM + +extern void xencons_suspend(void); +extern void xencons_resume(void); static void -xen_suspend(void *ignore) +xen_suspend() { int i, j, k, fpp; + unsigned long max_pfn; - extern void time_resume(void); - extern unsigned long max_pfn; - extern unsigned long *pfn_to_mfn_frame_list_list; - extern unsigned long *pfn_to_mfn_frame_list[]; - -#ifdef CONFIG_SMP -#error "do_suspend must be run cpu 0 - need to create separate thread" - cpumask_t prev_online_cpus; - int vcpu_prepare(int vcpu); -#endif - - int err = 0; - - PANIC_IF(smp_processor_id() != 0); - -#if defined(CONFIG_SMP) && !defined(CONFIG_HOTPLUG_CPU) - if (num_online_cpus() > 1) { - printk(KERN_WARNING "Can't suspend SMP guests " - "without CONFIG_HOTPLUG_CPU\n"); - return -EOPNOTSUPP; - } -#endif - - xenbus_suspend(); - -#ifdef CONFIG_SMP - lock_cpu_hotplug(); +#ifdef SMP + cpumask_t map; /* - * Take all other CPUs offline. We hold the hotplug semaphore to - * avoid other processes bringing up CPUs under our feet. + * Bind us to CPU 0 and stop any other VCPUs. */ - cpus_clear(prev_online_cpus); - while (num_online_cpus() > 1) { - for_each_online_cpu(i) { - if (i == 0) - continue; - unlock_cpu_hotplug(); - err = cpu_down(i); - lock_cpu_hotplug(); - if (err != 0) { - printk(KERN_CRIT "Failed to take all CPUs " - "down: %d.\n", err); - goto out_reenable_cpus; - } - cpu_set(i, prev_online_cpus); - } - } -#endif /* CONFIG_SMP */ + mtx_lock_spin(&sched_lock); + sched_bind(curthread, 0); + mtx_unlock_spin(&sched_lock); + KASSERT(PCPU_GET(cpuid) == 0, ("xen_suspend: not running on cpu 0")); + + map = PCPU_GET(other_cpus) & ~stopped_cpus; + if (map) + stop_cpus(map); +#endif - preempt_disable(); + if (DEVICE_SUSPEND(root_bus) != 0) { + printf("xen_suspend: device_suspend failed\n"); + if (map) + restart_cpus(map); + return; + } + local_irq_disable(); - __cli(); - preempt_enable(); -#ifdef SMP - unlock_cpu_hotplug(); -#endif + xencons_suspend(); gnttab_suspend(); - pmap_kremove(HYPERVISOR_shared_info); + max_pfn = HYPERVISOR_shared_info->arch.max_pfn; + + void *shared_info = HYPERVISOR_shared_info; + HYPERVISOR_shared_info = NULL; + pmap_kremove((vm_offset_t) shared_info); + PT_UPDATES_FLUSH(); - xen_start_info->store_mfn = mfn_to_pfn(xen_start_info->store_mfn); - xen_start_info->console.domU.mfn = mfn_to_pfn(xen_start_info->console.domU.mfn); + xen_start_info->store_mfn = MFNTOPFN(xen_start_info->store_mfn); + xen_start_info->console.domU.mfn = MFNTOPFN(xen_start_info->console.domU.mfn); /* * We'll stop somewhere inside this hypercall. When it returns, @@ -199,38 +214,33 @@ xen_suspend(void *ignore) */ HYPERVISOR_suspend(VTOMFN(xen_start_info)); - pmap_kenter_ma(HYPERVISOR_shared_info, xen_start_info->shared_info); - set_fixmap(FIX_SHARED_INFO, xen_start_info->shared_info); + pmap_kenter_ma((vm_offset_t) shared_info, xen_start_info->shared_info); + HYPERVISOR_shared_info = shared_info; -#if 0 - memset(empty_zero_page, 0, PAGE_SIZE); -#endif HYPERVISOR_shared_info->arch.pfn_to_mfn_frame_list_list = - VTOMFN(pfn_to_mfn_frame_list_list); + VTOMFN(xen_pfn_to_mfn_frame_list_list); fpp = PAGE_SIZE/sizeof(unsigned long); for (i = 0, j = 0, k = -1; i < max_pfn; i += fpp, j++) { if ((j % fpp) == 0) { k++; - pfn_to_mfn_frame_list_list[k] = - VTOMFN(pfn_to_mfn_frame_list[k]); + xen_pfn_to_mfn_frame_list_list[k] = + VTOMFN(xen_pfn_to_mfn_frame_list[k]); j = 0; } - pfn_to_mfn_frame_list[k][j] = - VTOMFN(&phys_to_machine_mapping[i]); + xen_pfn_to_mfn_frame_list[k][j] = + VTOMFN(&xen_phys_machine[i]); } HYPERVISOR_shared_info->arch.max_pfn = max_pfn; gnttab_resume(); - irq_resume(); - - time_resume(); - - __sti(); - + cpu_initclocks(); + local_irq_enable(); xencons_resume(); + printf("UP\n"); + #ifdef CONFIG_SMP for_each_cpu(i) vcpu_prepare(i); @@ -240,21 +250,13 @@ xen_suspend(void *ignore) * Only resume xenbus /after/ we've prepared our VCPUs; otherwise * the VCPU hotplug callback can race with our vcpu_prepare */ - xenbus_resume(); + DEVICE_RESUME(root_bus); -#ifdef CONFIG_SMP - out_reenable_cpus: - for_each_cpu_mask(i, prev_online_cpus) { - j = cpu_up(i); - if ((j != 0) && !cpu_online(i)) { - printk(KERN_CRIT "Failed to bring cpu " - "%d back up (%d).\n", - i, j); - err = j; - } - } +#ifdef SMP + sched_unbind(curthread); + if (map) + restart_cpus(map); #endif - return err; } -#endif /* notyet */ +#endif Modified: user/dfr/xenhvm/6/sys/xen/xenbus/xenbus_probe.c ============================================================================== --- user/dfr/xenhvm/6/sys/xen/xenbus/xenbus_probe.c Tue Dec 9 17:47:00 2008 (r185802) +++ user/dfr/xenhvm/6/sys/xen/xenbus/xenbus_probe.c Tue Dec 9 17:47:05 2008 (r185803) @@ -389,28 +389,90 @@ xenbus_attach(device_t dev) return (0); } -static void +static int xenbus_suspend(device_t dev) { + int error; + DPRINTK(""); - panic("implement me"); -#if 0 - bus_for_each_dev(&xenbus_frontend.bus, NULL, NULL, suspend_dev); - bus_for_each_dev(&xenbus_backend.bus, NULL, NULL, suspend_dev); -#endif + + error = bus_generic_suspend(dev); + if (error) + return (error); + xs_suspend(); + + return (0); } -static void +static int xenbus_resume(device_t dev) { + device_t *kids; + struct xenbus_device_ivars *ivars; + int i, count, error; + char *statepath; + xb_init_comms(); xs_resume(); - panic("implement me"); + + /* + * We must re-examine each device and find the new path for + * its backend. + */ + if (device_get_children(dev, &kids, &count) == 0) { + for (i = 0; i < count; i++) { + if (device_get_state(kids[i]) == DS_NOTPRESENT) + continue; + + ivars = device_get_ivars(kids[i]); + + unregister_xenbus_watch( + &ivars->xd_otherend_watch); + ivars->xd_state = XenbusStateInitialising; + + /* + * Find the new backend details and + * re-register our watch. + */ + free(ivars->xd_otherend_path, M_DEVBUF); + error = xenbus_gather(XBT_NIL, ivars->xd_node, + "backend-id", "%i", &ivars->xd_otherend_id, + "backend", NULL, &ivars->xd_otherend_path, + NULL); + if (error) + return (error); + + DEVICE_RESUME(kids[i]); + + statepath = malloc(strlen(ivars->xd_otherend_path) + + strlen("/state") + 1, M_DEVBUF, M_WAITOK); + sprintf(statepath, "%s/state", ivars->xd_otherend_path); + + free(ivars->xd_otherend_watch.node, M_DEVBUF); + ivars->xd_otherend_watch.node = statepath; + register_xenbus_watch( + &ivars->xd_otherend_watch); + #if 0 *** DIFF OUTPUT TRUNCATED AT 1000 LINES ***