Date: Wed, 19 Oct 2016 05:08:46 +0000 (UTC) From: Sepherosa Ziehau <sephe@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-stable@freebsd.org, svn-src-stable-11@freebsd.org Subject: svn commit: r307603 - stable/11/sys/dev/hyperv/netvsc Message-ID: <201610190508.u9J58kKo066439@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: sephe Date: Wed Oct 19 05:08:46 2016 New Revision: 307603 URL: https://svnweb.freebsd.org/changeset/base/307603 Log: MFC 306390-306392 306390 hyperv/hn: Suspend and resume the backend properly upon MTU change. Suspend: - Prevent the backend from being touched on TX path. - Clear the RNDIS RX filter, and wait for RX to drain. - Make sure that NVS see the chimney sending buffer and RXBUF disconnection, before unlink these buffers from the channel. Resume: - Reconfigure the RNDIS filter. - Allow TX path to work on the backend. - Kick start the TX eof task, in case the OACTIVE is set. This fixes various panics, when the interface has traffic and MTU is being changed. Sponsored by: Microsoft Differential Revision: https://reviews.freebsd.org/D8046 306391 hyperv/hn: Reorganize the synthetic parts detach. Sponsored by: Microsoft Differential Revision: https://reviews.freebsd.org/D8047 306392 hyperv/hn: Reorder the comment a little bit. Sponsored by: Microsoft Differential Revision: https://reviews.freebsd.org/D8048 Modified: stable/11/sys/dev/hyperv/netvsc/hv_net_vsc.c stable/11/sys/dev/hyperv/netvsc/hv_net_vsc.h stable/11/sys/dev/hyperv/netvsc/hv_netvsc_drv_freebsd.c stable/11/sys/dev/hyperv/netvsc/hv_rndis_filter.c stable/11/sys/dev/hyperv/netvsc/hv_rndis_filter.h stable/11/sys/dev/hyperv/netvsc/if_hnvar.h Directory Properties: stable/11/ (props changed) Modified: stable/11/sys/dev/hyperv/netvsc/hv_net_vsc.c ============================================================================== --- stable/11/sys/dev/hyperv/netvsc/hv_net_vsc.c Wed Oct 19 05:03:46 2016 (r307602) +++ stable/11/sys/dev/hyperv/netvsc/hv_net_vsc.c Wed Oct 19 05:08:46 2016 (r307603) @@ -348,6 +348,16 @@ hn_nvs_disconn_rxbuf(struct hn_softc *sc return (error); } sc->hn_flags &= ~HN_FLAG_RXBUF_CONNECTED; + + /* + * Wait for the hypervisor to receive this NVS request. + */ + while (!vmbus_chan_tx_empty(sc->hn_prichan)) + pause("waittx", 1); + /* + * Linger long enough for NVS to disconnect RXBUF. + */ + pause("lingtx", (200 * hz) / 1000); } if (sc->hn_rxbuf_gpadl != 0) { @@ -389,6 +399,17 @@ hn_nvs_disconn_chim(struct hn_softc *sc) return (error); } sc->hn_flags &= ~HN_FLAG_CHIM_CONNECTED; + + /* + * Wait for the hypervisor to receive this NVS request. + */ + while (!vmbus_chan_tx_empty(sc->hn_prichan)) + pause("waittx", 1); + /* + * Linger long enough for NVS to disconnect chimney + * sending buffer. + */ + pause("lingtx", (200 * hz) / 1000); } if (sc->hn_chim_gpadl != 0) { @@ -597,27 +618,15 @@ hn_nvs_attach(struct hn_softc *sc, int m return (0); } -/* - * Net VSC disconnect from VSP - */ -static void -hv_nv_disconnect_from_vsp(struct hn_softc *sc) +void +hn_nvs_detach(struct hn_softc *sc) { + + /* NOTE: there are no requests to stop the NVS. */ hn_nvs_disconn_rxbuf(sc); hn_nvs_disconn_chim(sc); } -/* - * Net VSC on device remove - */ -int -hv_nv_on_device_remove(struct hn_softc *sc) -{ - - hv_nv_disconnect_from_vsp(sc); - return (0); -} - void hn_nvs_sent_xact(struct hn_send_ctx *sndc, struct hn_softc *sc __unused, struct vmbus_channel *chan __unused, Modified: stable/11/sys/dev/hyperv/netvsc/hv_net_vsc.h ============================================================================== --- stable/11/sys/dev/hyperv/netvsc/hv_net_vsc.h Wed Oct 19 05:03:46 2016 (r307602) +++ stable/11/sys/dev/hyperv/netvsc/hv_net_vsc.h Wed Oct 19 05:08:46 2016 (r307603) @@ -178,6 +178,7 @@ struct hn_tx_ring { bus_dma_tag_t hn_tx_data_dtag; uint64_t hn_csum_assist; + int hn_suspended; int hn_gpa_cnt; struct vmbus_gpa hn_gpa[NETVSC_PACKET_MAXPAGE]; @@ -268,8 +269,6 @@ extern int hv_promisc_mode; struct hn_send_ctx; void netvsc_linkstatus_callback(struct hn_softc *sc, uint32_t status); -int hn_nvs_attach(struct hn_softc *sc, int mtu); -int hv_nv_on_device_remove(struct hn_softc *sc); int hv_nv_on_send(struct vmbus_channel *chan, uint32_t rndis_mtype, struct hn_send_ctx *sndc, struct vmbus_gpa *gpa, int gpa_cnt); Modified: stable/11/sys/dev/hyperv/netvsc/hv_netvsc_drv_freebsd.c ============================================================================== --- stable/11/sys/dev/hyperv/netvsc/hv_netvsc_drv_freebsd.c Wed Oct 19 05:03:46 2016 (r307602) +++ stable/11/sys/dev/hyperv/netvsc/hv_netvsc_drv_freebsd.c Wed Oct 19 05:08:46 2016 (r307603) @@ -348,6 +348,11 @@ static void hn_detach_allchans(struct hn static void hn_chan_callback(struct vmbus_channel *chan, void *xrxr); static void hn_set_ring_inuse(struct hn_softc *, int); static int hn_synth_attach(struct hn_softc *, int); +static void hn_synth_detach(struct hn_softc *); +static bool hn_tx_ring_pending(struct hn_tx_ring *); +static void hn_suspend(struct hn_softc *); +static void hn_resume(struct hn_softc *); +static void hn_tx_ring_qflush(struct hn_tx_ring *); static void hn_nvs_handle_notify(struct hn_softc *sc, const struct vmbus_chanpkt_hdr *pkt); @@ -740,29 +745,19 @@ failed: } /* - * Standard detach entry point + * TODO: Use this for error handling on attach path. */ static int netvsc_detach(device_t dev) { struct hn_softc *sc = device_get_softc(dev); - if (bootverbose) - printf("netvsc_detach\n"); - - /* - * XXXKYS: Need to clean up all our - * driver state; this is the driver - * unloading. - */ + /* TODO: ether_ifdetach */ - /* - * XXXKYS: Need to stop outgoing traffic and unregister - * the netdevice. - */ - - hv_rf_on_device_remove(sc); - hn_detach_allchans(sc); + HN_LOCK(sc); + /* TODO: hn_stop */ + hn_synth_detach(sc); + HN_UNLOCK(sc); hn_stop_tx_tasks(sc); @@ -775,6 +770,8 @@ netvsc_detach(device_t dev) vmbus_xact_ctx_destroy(sc->hn_xact); HN_LOCK_DESTROY(sc); + + /* TODO: if_free */ return (0); } @@ -905,6 +902,23 @@ hn_txdesc_hold(struct hn_txdesc *txd) atomic_add_int(&txd->refs, 1); } +static bool +hn_tx_ring_pending(struct hn_tx_ring *txr) +{ + bool pending = false; + +#ifndef HN_USE_TXDESC_BUFRING + mtx_lock_spin(&txr->hn_txlist_spin); + if (txr->hn_txdesc_avail != txr->hn_txdesc_cnt) + pending = true; + mtx_unlock_spin(&txr->hn_txlist_spin); +#else + if (!buf_ring_full(txr->hn_txdesc_br)) + pending = true; +#endif + return (pending); +} + static __inline void hn_txeof(struct hn_tx_ring *txr) { @@ -1241,6 +1255,9 @@ hn_start_locked(struct hn_tx_ring *txr, KASSERT(txr == &sc->hn_tx_ring[0], ("not the first TX ring")); mtx_assert(&txr->hn_tx_lock, MA_OWNED); + if (__predict_false(txr->hn_suspended)) + return 0; + if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != IFF_DRV_RUNNING) return 0; @@ -1627,23 +1644,17 @@ hn_ioctl(struct ifnet *ifp, u_long cmd, hn_set_lro_lenlim(sc, HN_LRO_LENLIM_MIN(ifp)); #endif - /* We must remove and add back the device to cause the new - * MTU to take effect. This includes tearing down, but not - * deleting the channel, then bringing it back up. - */ - error = hv_rf_on_device_remove(sc); - if (error) { - HN_UNLOCK(sc); - break; - } + if (ifp->if_drv_flags & IFF_DRV_RUNNING) + hn_suspend(sc); /* - * Detach all of the channels. + * Detach the synthetics parts, i.e. NVS and RNDIS. */ - hn_detach_allchans(sc); + hn_synth_detach(sc); /* - * Attach the synthetic parts, i.e. NVS and RNDIS. + * Reattach the synthetic parts, i.e. NVS and RNDIS, + * with the new MTU setting. * XXX check error. */ hn_synth_attach(sc, ifr->ifr_mtu); @@ -1651,7 +1662,9 @@ hn_ioctl(struct ifnet *ifp, u_long cmd, if (sc->hn_tx_ring[0].hn_chim_size > sc->hn_chim_szmax) hn_set_chim_size(sc, sc->hn_chim_szmax); - hn_init_locked(sc); + /* All done! Resume now. */ + if (ifp->if_drv_flags & IFF_DRV_RUNNING) + hn_resume(sc); HN_UNLOCK(sc); break; @@ -2984,6 +2997,9 @@ hn_xmit(struct hn_tx_ring *txr, int len) KASSERT(hn_use_if_start == 0, ("hn_xmit is called, when if_start is enabled")); + if (__predict_false(txr->hn_suspended)) + return 0; + if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0 || txr->hn_oactive) return 0; @@ -3070,20 +3086,24 @@ do_sched: } static void +hn_tx_ring_qflush(struct hn_tx_ring *txr) +{ + struct mbuf *m; + + mtx_lock(&txr->hn_tx_lock); + while ((m = buf_ring_dequeue_sc(txr->hn_mbuf_br)) != NULL) + m_freem(m); + mtx_unlock(&txr->hn_tx_lock); +} + +static void hn_xmit_qflush(struct ifnet *ifp) { struct hn_softc *sc = ifp->if_softc; int i; - for (i = 0; i < sc->hn_tx_ring_inuse; ++i) { - struct hn_tx_ring *txr = &sc->hn_tx_ring[i]; - struct mbuf *m; - - mtx_lock(&txr->hn_tx_lock); - while ((m = buf_ring_dequeue_sc(txr->hn_mbuf_br)) != NULL) - m_freem(m); - mtx_unlock(&txr->hn_tx_lock); - } + for (i = 0; i < sc->hn_tx_ring_inuse; ++i) + hn_tx_ring_qflush(&sc->hn_tx_ring[i]); if_qflush(ifp); } @@ -3484,6 +3504,26 @@ back: return (0); } +/* + * NOTE: + * The interface must have been suspended though hn_suspend(), before + * this function get called. + */ +static void +hn_synth_detach(struct hn_softc *sc) +{ + HN_LOCK_ASSERT(sc); + + /* Detach the RNDIS first. */ + hn_rndis_detach(sc); + + /* Detach NVS. */ + hn_nvs_detach(sc); + + /* Detach all of the channels. */ + hn_detach_allchans(sc); +} + static void hn_set_ring_inuse(struct hn_softc *sc, int ring_cnt) { @@ -3503,6 +3543,113 @@ hn_set_ring_inuse(struct hn_softc *sc, i } static void +hn_rx_drain(struct vmbus_channel *chan) +{ + + while (!vmbus_chan_rx_empty(chan) || !vmbus_chan_tx_empty(chan)) + pause("waitch", 1); + vmbus_chan_intr_drain(chan); +} + +static void +hn_suspend(struct hn_softc *sc) +{ + struct vmbus_channel **subch = NULL; + int i, nsubch; + + HN_LOCK_ASSERT(sc); + + /* + * Suspend TX. + */ + for (i = 0; i < sc->hn_tx_ring_inuse; ++i) { + struct hn_tx_ring *txr = &sc->hn_tx_ring[i]; + + mtx_lock(&txr->hn_tx_lock); + txr->hn_suspended = 1; + mtx_unlock(&txr->hn_tx_lock); + /* No one is able send more packets now. */ + + /* Wait for all pending sends to finish. */ + while (hn_tx_ring_pending(txr)) + pause("hnwtx", 1 /* 1 tick */); + } + + /* + * Disable RX. + */ + hv_rf_on_close(sc); + + /* + * Give RNDIS enough time to flush all pending data packets. + */ + pause("waitrx", (200 * hz) / 1000); + + /* + * Drain RX/TX bufrings and interrupts. + */ + nsubch = sc->hn_rx_ring_inuse - 1; + if (nsubch > 0) + subch = vmbus_subchan_get(sc->hn_prichan, nsubch); + + if (subch != NULL) { + for (i = 0; i < nsubch; ++i) + hn_rx_drain(subch[i]); + } + hn_rx_drain(sc->hn_prichan); + + if (subch != NULL) + vmbus_subchan_rel(subch, nsubch); +} + +static void +hn_resume(struct hn_softc *sc) +{ + struct hn_tx_ring *txr; + int i; + + HN_LOCK_ASSERT(sc); + + /* + * Re-enable RX. + */ + hv_rf_on_open(sc); + + /* + * Make sure to clear suspend status on "all" TX rings, + * since hn_tx_ring_inuse can be changed after hn_suspend(). + */ + for (i = 0; i < sc->hn_tx_ring_cnt; ++i) { + txr = &sc->hn_tx_ring[i]; + + mtx_lock(&txr->hn_tx_lock); + txr->hn_suspended = 0; + mtx_unlock(&txr->hn_tx_lock); + } + + if (!hn_use_if_start) { + /* + * Flush unused drbrs, since hn_tx_ring_inuse may be + * reduced. + */ + for (i = sc->hn_tx_ring_inuse; i < sc->hn_tx_ring_cnt; ++i) + hn_tx_ring_qflush(&sc->hn_tx_ring[i]); + } + + /* + * Kick start TX. + */ + for (i = 0; i < sc->hn_tx_ring_inuse; ++i) { + txr = &sc->hn_tx_ring[i]; + /* + * Use txeof task, so that any pending oactive can be + * cleared properly. + */ + taskqueue_enqueue(txr->hn_tx_taskq, &txr->hn_txeof_task); + } +} + +static void hn_nvs_handle_notify(struct hn_softc *sc, const struct vmbus_chanpkt_hdr *pkt) { const struct hn_nvs_hdr *hdr; Modified: stable/11/sys/dev/hyperv/netvsc/hv_rndis_filter.c ============================================================================== --- stable/11/sys/dev/hyperv/netvsc/hv_rndis_filter.c Wed Oct 19 05:03:46 2016 (r307602) +++ stable/11/sys/dev/hyperv/netvsc/hv_rndis_filter.c Wed Oct 19 05:08:46 2016 (r307603) @@ -957,11 +957,8 @@ done: return (error); } -/* - * RNDIS filter halt device - */ static int -hv_rf_halt_device(struct hn_softc *sc) +hn_rndis_halt(struct hn_softc *sc) { struct vmbus_xact *xact; struct rndis_halt_req *halt; @@ -1008,21 +1005,12 @@ hn_rndis_attach(struct hn_softc *sc) return (0); } -/* - * RNDIS filter on device remove - */ -int -hv_rf_on_device_remove(struct hn_softc *sc) +void +hn_rndis_detach(struct hn_softc *sc) { - int ret; - - /* Halt and release the rndis device */ - ret = hv_rf_halt_device(sc); - - /* Pass control to inner driver to remove the device */ - ret |= hv_nv_on_device_remove(sc); - return (ret); + /* Halt the RNDIS. */ + hn_rndis_halt(sc); } /* Modified: stable/11/sys/dev/hyperv/netvsc/hv_rndis_filter.h ============================================================================== --- stable/11/sys/dev/hyperv/netvsc/hv_rndis_filter.h Wed Oct 19 05:03:46 2016 (r307602) +++ stable/11/sys/dev/hyperv/netvsc/hv_rndis_filter.h Wed Oct 19 05:08:46 2016 (r307603) @@ -43,7 +43,6 @@ struct hn_rx_ring; void hv_rf_on_receive(struct hn_softc *sc, struct hn_rx_ring *rxr, const void *data, int dlen); void hv_rf_channel_rollup(struct hn_rx_ring *rxr, struct hn_tx_ring *txr); -int hv_rf_on_device_remove(struct hn_softc *sc); int hv_rf_on_open(struct hn_softc *sc); int hv_rf_on_close(struct hn_softc *sc); Modified: stable/11/sys/dev/hyperv/netvsc/if_hnvar.h ============================================================================== --- stable/11/sys/dev/hyperv/netvsc/if_hnvar.h Wed Oct 19 05:03:46 2016 (r307602) +++ stable/11/sys/dev/hyperv/netvsc/if_hnvar.h Wed Oct 19 05:08:46 2016 (r307603) @@ -118,6 +118,7 @@ uint32_t hn_chim_alloc(struct hn_softc * void hn_chim_free(struct hn_softc *sc, uint32_t chim_idx); int hn_rndis_attach(struct hn_softc *sc); +void hn_rndis_detach(struct hn_softc *sc); int hn_rndis_conf_rss(struct hn_softc *sc, uint16_t flags); void *hn_rndis_pktinfo_append(struct rndis_packet_msg *, size_t pktsize, size_t pi_dlen, uint32_t pi_type); @@ -127,6 +128,7 @@ int hn_rndis_get_linkstatus(struct hn_s uint32_t *link_status); int hn_nvs_attach(struct hn_softc *sc, int mtu); +void hn_nvs_detach(struct hn_softc *sc); int hn_nvs_alloc_subchans(struct hn_softc *sc, int *nsubch); void hn_nvs_sent_xact(struct hn_send_ctx *sndc, struct hn_softc *sc, struct vmbus_channel *chan, const void *data, int dlen);
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201610190508.u9J58kKo066439>