From owner-p4-projects@FreeBSD.ORG Thu Aug 2 23:14:37 2007 Return-Path: Delivered-To: p4-projects@freebsd.org Received: by hub.freebsd.org (Postfix, from userid 32767) id 1A2C616A419; Thu, 2 Aug 2007 23:14:37 +0000 (UTC) Delivered-To: perforce@FreeBSD.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:4f8:fff6::34]) by hub.freebsd.org (Postfix) with ESMTP id B42BF16A418 for ; Thu, 2 Aug 2007 23:14:36 +0000 (UTC) (envelope-from zec@FreeBSD.org) Received: from repoman.freebsd.org (repoman.freebsd.org [IPv6:2001:4f8:fff6::29]) by mx1.freebsd.org (Postfix) with ESMTP id A530C13C428 for ; Thu, 2 Aug 2007 23:14:36 +0000 (UTC) (envelope-from zec@FreeBSD.org) Received: from repoman.freebsd.org (localhost [127.0.0.1]) by repoman.freebsd.org (8.14.1/8.14.1) with ESMTP id l72NEa83087582 for ; Thu, 2 Aug 2007 23:14:36 GMT (envelope-from zec@FreeBSD.org) Received: (from perforce@localhost) by repoman.freebsd.org (8.14.1/8.14.1/Submit) id l72NEaVU087579 for perforce@freebsd.org; Thu, 2 Aug 2007 23:14:36 GMT (envelope-from zec@FreeBSD.org) Date: Thu, 2 Aug 2007 23:14:36 GMT Message-Id: <200708022314.l72NEaVU087579@repoman.freebsd.org> X-Authentication-Warning: repoman.freebsd.org: perforce set sender to zec@FreeBSD.org using -f From: Marko Zec To: Perforce Change Reviews Cc: Subject: PERFORCE change 124550 for review X-BeenThere: p4-projects@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: p4 projects tree changes List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 02 Aug 2007 23:14:37 -0000 http://perforce.freebsd.org/chv.cgi?CH=124550 Change 124550 by zec@zec_tpx32 on 2007/08/02 23:14:03 Experimentation with enforcing initialization ordering of vnet modules by specifiying module interdepencencies. The vnet module registration process was contemplated so that it should closely resemble the initialization ordering of networking subsystems in the standard kernel. However, it turns out that in some circumstances we might need more strict ordering enforcement on a virtualized system, given that for some subsystems (e.g. netgraph) the initialization sequence can vary significantly depending on whether the code is statically compiled in the kernel, or is kldloaded at run time. The ordering problem is perhaps more pronounced during cleanups of vnet instances, for which our networking code was never really designed. Hence, each vnet module can now specify another module for which it expects to be handled earlier during the vnet instatiation process than itself. The module registration handler will check whether the prerequisite module is already registered, and if not, postpone the registration of the requested module until the prerequisite shows up. Only the time can tell whether such a mechanism makes real sense... While here, in effect revert change 124477 to ng_base.c, given that now netgraph initialization point will be determined through the new dependency enforcement mechanisms. Affected files ... .. //depot/projects/vimage/src/sys/kern/kern_vimage.c#31 edit .. //depot/projects/vimage/src/sys/kern/uipc_domain.c#9 edit .. //depot/projects/vimage/src/sys/net/if.c#18 edit .. //depot/projects/vimage/src/sys/net/if_gif.c#8 edit .. //depot/projects/vimage/src/sys/net/if_loop.c#16 edit .. //depot/projects/vimage/src/sys/net/route.c#10 edit .. //depot/projects/vimage/src/sys/netgraph/ng_base.c#17 edit .. //depot/projects/vimage/src/sys/netinet/if_ether.c#13 edit .. //depot/projects/vimage/src/sys/netinet/ip_fw2.c#20 edit .. //depot/projects/vimage/src/sys/netinet/ip_input.c#20 edit .. //depot/projects/vimage/src/sys/netinet6/ip6_input.c#17 edit .. //depot/projects/vimage/src/sys/netipsec/ipsec.c#13 edit .. //depot/projects/vimage/src/sys/netipsec/xform_ah.c#9 edit .. //depot/projects/vimage/src/sys/netipsec/xform_esp.c#8 edit .. //depot/projects/vimage/src/sys/netipsec/xform_ipcomp.c#7 edit .. //depot/projects/vimage/src/sys/netipsec/xform_ipip.c#10 edit .. //depot/projects/vimage/src/sys/sys/vimage.h#29 edit Differences ... ==== //depot/projects/vimage/src/sys/kern/kern_vimage.c#31 (text+ko) ==== @@ -69,6 +69,7 @@ MALLOC_DEFINE(M_VCPU, "vcpu", "cpu resource control block"); static void vi_destroy(struct vimage *); +static void vnet_mod_complete_registration(struct vnet_modlink *); static int vnet_mod_constructor(struct vnet_modlink *); static int vnet_mod_destructor(struct vnet_modlink *); @@ -125,6 +126,7 @@ static int last_vi_id = 0; static TAILQ_HEAD(vnet_modlink_head, vnet_modlink) vnet_modlink_head; +static TAILQ_HEAD(vnet_modpending_head, vnet_modlink) vnet_modpending_head; void vnet_mod_register(vmi) const struct vnet_modinfo *vmi; @@ -137,13 +139,13 @@ const void *iarg; const char *iname; { - struct vnet_modlink *vml; + struct vnet_modlink *vml, *vml_iter; /* Do not register the same module instance more than once */ - TAILQ_FOREACH(vml, &vnet_modlink_head, vml_mod_le) - if (vml->vml_modinfo == vmi && vml->vml_iarg == iarg) + TAILQ_FOREACH(vml_iter, &vnet_modlink_head, vml_mod_le) + if (vml_iter->vml_modinfo == vmi && vml_iter->vml_iarg == iarg) break; - if (vml != NULL) + if (vml_iter != NULL) panic("attempt to register an already registered vnet module"); vml = vi_malloc(sizeof(struct vnet_modlink), M_VIMAGE, M_NOWAIT); @@ -158,11 +160,52 @@ vml->vml_modinfo = vmi; vml->vml_iarg = iarg; vml->vml_iname = iname; + + /* Check whether the module we depend on is already registered */ + if (vmi->vmi_dependson != VNET_MOD_NONE) { + TAILQ_FOREACH(vml_iter, &vnet_modlink_head, vml_mod_le) + if (vml_iter->vml_modinfo->vmi_id == + vmi->vmi_dependson) + break; /* Depencency found, we are done */ + if (vml_iter == NULL) { +#ifdef DEBUG_ORDERING + printf("dependency %d missing for vnet mod %s, postponing registration\n", vmi->vmi_dependson, vmi->vmi_name); +#endif /* DEBUG_ORDERING */ + TAILQ_INSERT_TAIL(&vnet_modpending_head, vml, + vml_mod_le); + return; + } + } + + vnet_mod_complete_registration(vml); +} + +void vnet_mod_complete_registration(vml) +struct vnet_modlink *vml; +{ + struct vnet_modlink *vml_iter; + TAILQ_INSERT_TAIL(&vnet_modlink_head, vml, vml_mod_le); VNET_ITERLOOP_BEGIN_QUIET(); vnet_mod_constructor(vml); VNET_ITERLOOP_END(); + + /* Check for pending modules depending on us */ + do { + TAILQ_FOREACH(vml_iter, &vnet_modpending_head, vml_mod_le) + if (vml_iter->vml_modinfo->vmi_dependson == + vml->vml_modinfo->vmi_id) + break; + if (vml_iter != NULL) { +#ifdef DEBUG_ORDERING + printf("vnet mod %s now registering, dependency %d loaded\n", vml_iter->vml_modinfo->vmi_name, vml->vml_modinfo->vmi_id); +#endif /* DEBUG_ORDERING */ + TAILQ_REMOVE(&vnet_modpending_head, vml_iter, + vml_mod_le); + vnet_mod_complete_registration(vml_iter); + } + } while (vml_iter != NULL); } void vnet_mod_deregister(vmi) @@ -675,6 +718,7 @@ /* vnet module list is both forward and reverse traversable */ TAILQ_INIT(&vnet_modlink_head); + TAILQ_INIT(&vnet_modpending_head); LIST_INIT(&vimage_head); LIST_INIT(&vnet_head); @@ -700,7 +744,19 @@ static void vi_init_done(void *unused) { + struct vnet_modlink *vml_iter; + curvnet = NULL; + + if (TAILQ_EMPTY(&vnet_modpending_head)) + return; + + printf("vnet modules with unresolved dependencies:\n"); + TAILQ_FOREACH(vml_iter, &vnet_modpending_head, vml_mod_le) + printf(" %s depending on %d:\n", + vml_iter->vml_modinfo->vmi_name, + vml_iter->vml_modinfo->vmi_dependson); + panic("going nowhere without my vnet modules!"); } SYSINIT(vimage, SI_SUB_VIMAGE, SI_ORDER_FIRST, vi_init, NULL) ==== //depot/projects/vimage/src/sys/kern/uipc_domain.c#9 (text+ko) ==== @@ -108,7 +108,8 @@ .pru_sopoll = pru_sopoll_notsupp, }; -VNET_MOD_DECLARE_STATELESS(DOMAIN, domain, net_init_domain, net_detach_domain) +VNET_MOD_DECLARE_STATELESS(DOMAIN, domain, net_init_domain, net_detach_domain, + NET) static void protosw_init(struct protosw *pr) ==== //depot/projects/vimage/src/sys/net/if.c#18 (text+ko) ==== @@ -170,9 +170,10 @@ VNET_SYMMAP(net, rttrash), VNET_SYMMAP_END }; +#endif -VNET_MOD_DECLARE(NET, net, vnet_net_symmap, vnet_net_iattach, vnet_net_idetach) -#endif +VNET_MOD_DECLARE(NET, net, vnet_net_iattach, vnet_net_idetach, + NONE, vnet_net_symmap) /* * System initialization ==== //depot/projects/vimage/src/sys/net/if_gif.c#8 (text+ko) ==== @@ -144,7 +144,7 @@ SYSCTL_V_INT(V_NET, vnet_gif, _net_link_gif, OID_AUTO, parallel_tunnels, CTLFLAG_RW, parallel_tunnels, 0, "Allow parallel tunnels?"); -VNET_MOD_DECLARE(GIF, gif, NULL, vnet_gif_iattach, NULL) +VNET_MOD_DECLARE(GIF, gif, NULL, vnet_gif_iattach, NET, NULL) static int gif_clone_create(ifc, unit, params) ==== //depot/projects/vimage/src/sys/net/if_loop.c#16 (text+ko) ==== @@ -180,7 +180,8 @@ return (0); } -VNET_MOD_DECLARE_STATELESS(LOIF, loif, vnet_loif_iattach, vnet_loif_idetach) +VNET_MOD_DECLARE_STATELESS(LOIF, loif, vnet_loif_iattach, vnet_loif_idetach, + NET) static int vnet_loif_iattach(unused) const void *unused; ==== //depot/projects/vimage/src/sys/net/route.c#10 (text+ko) ==== @@ -80,7 +80,8 @@ */ #define RNTORT(p) ((struct rtentry *)(p)) -VNET_MOD_DECLARE_STATELESS(RTABLE, rtable, rtable_init, rtable_idetach) +VNET_MOD_DECLARE_STATELESS(RTABLE, rtable, rtable_init, rtable_idetach, + NET) static int rtable_init(unused) ==== //depot/projects/vimage/src/sys/netgraph/ng_base.c#17 (text+ko) ==== @@ -251,12 +251,10 @@ static vnet_attach_fn vnet_netgraph_iattach; #ifdef VIMAGE static vnet_detach_fn vnet_netgraph_idetach; -static void netgraph_register(void *); -SYSINIT(netgraph, SI_SUB_PROTO_END, SI_ORDER_FIRST, netgraph_register, NULL) #endif /* VIMAGE */ -VNET_MOD_DECLARE(NETGRAPH, netgraph, NULL, vnet_netgraph_iattach, - vnet_netgraph_idetach) +VNET_MOD_DECLARE(NETGRAPH, netgraph, vnet_netgraph_iattach, + vnet_netgraph_idetach, LOIF, NULL) #ifdef NETGRAPH_DEBUG /*----------------------------------------------*/ /* @@ -3191,7 +3189,9 @@ uma_zone_set_max(ng_qzone, maxalloc); netisr_register(NETISR_NETGRAPH, (netisr_t *)ngintr, NULL, NETISR_MPSAFE); -#ifndef VIMAGE +#ifdef VIMAGE + vnet_mod_register(&vnet_netgraph_modinfo); +#else vnet_netgraph_iattach(NULL); #endif /* !VIMAGE */ break; @@ -3239,11 +3239,6 @@ return 0; } - -void netgraph_register(void *unused) -{ - vnet_mod_register(&vnet_netgraph_modinfo); -} #endif /* VIMAGE */ static moduledata_t netgraph_mod = { ==== //depot/projects/vimage/src/sys/netinet/if_ether.c#13 (text+ko) ==== @@ -131,7 +131,7 @@ static void in_arpinput(struct mbuf *); #endif -VNET_MOD_DECLARE_STATELESS(ARP, arp, arp_iattach, NULL) +VNET_MOD_DECLARE_STATELESS(ARP, arp, arp_iattach, NULL, INET) /* * Timeout routine. ==== //depot/projects/vimage/src/sys/netinet/ip_fw2.c#20 (text+ko) ==== @@ -115,7 +115,7 @@ static int vnet_ipfw_iattach(const void *); static int vnet_ipfw_idetach(const void *); -VNET_MOD_DECLARE(IPFW, ipfw, NULL, vnet_ipfw_iattach, vnet_ipfw_idetach) +VNET_MOD_DECLARE(IPFW, ipfw, vnet_ipfw_iattach, vnet_ipfw_idetach, INET, NULL) /* * set_disable contains one bit per set value (0..31). ==== //depot/projects/vimage/src/sys/netinet/ip_input.c#20 (text+ko) ==== @@ -215,7 +215,7 @@ #ifdef VIMAGE static void vnet_inet_register(void); -VNET_MOD_DECLARE(INET, inet, NULL, NULL, NULL) +VNET_MOD_DECLARE(INET, inet, NULL, NULL, NET, NULL) static void vnet_inet_register() { ==== //depot/projects/vimage/src/sys/netinet6/ip6_input.c#17 (text+ko) ==== @@ -148,7 +148,7 @@ #ifdef VIMAGE static void vnet_inet6_register(void); -VNET_MOD_DECLARE(INET6, inet6, NULL, NULL, NULL) +VNET_MOD_DECLARE(INET6, inet6, NULL, NULL, INET, NULL) static void vnet_inet6_register() { ==== //depot/projects/vimage/src/sys/netipsec/ipsec.c#13 (text+ko) ==== @@ -223,9 +223,8 @@ ipsecstats, CTLFLAG_RD, ipsec6stat, ipsecstat, ""); #endif /* INET6 */ -#ifdef VIMAGE -VNET_MOD_DECLARE(IPSEC, ipsec, NULL, vnet_ipsec_iattach, vnet_ipsec_idetach) -#endif +VNET_MOD_DECLARE(IPSEC, ipsec, vnet_ipsec_iattach, vnet_ipsec_idetach, + INET, NULL) static int ipsec4_setspidx_inpcb __P((struct mbuf *, struct inpcb *pcb)); #ifdef INET6 ==== //depot/projects/vimage/src/sys/netipsec/xform_ah.c#9 (text+ko) ==== @@ -106,7 +106,7 @@ static int ah_iattach(const void *); -VNET_MOD_DECLARE_STATELESS(AH, ah, ah_iattach, NULL) +VNET_MOD_DECLARE_STATELESS(AH, ah, ah_iattach, NULL, IPSEC) static unsigned char ipseczeroes[256]; /* larger than an ip6 extension hdr */ ==== //depot/projects/vimage/src/sys/netipsec/xform_esp.c#8 (text+ko) ==== @@ -96,7 +96,7 @@ static int esp_iattach(const void *); -VNET_MOD_DECLARE_STATELESS(ESP, esp, esp_iattach, NULL) +VNET_MOD_DECLARE_STATELESS(ESP, esp, esp_iattach, NULL, IPSEC) /* * NB: this is public for use by the PF_KEY support. ==== //depot/projects/vimage/src/sys/netipsec/xform_ipcomp.c#7 (text+ko) ==== @@ -82,7 +82,7 @@ static int ipcomp_iattach(const void *); -VNET_MOD_DECLARE_STATELESS(IPCOMP, ipcomp, ipcomp_iattach, NULL) +VNET_MOD_DECLARE_STATELESS(IPCOMP, ipcomp, ipcomp_iattach, NULL, IPSEC) static int ipcomp_input_cb(struct cryptop *crp); static int ipcomp_output_cb(struct cryptop *crp); ==== //depot/projects/vimage/src/sys/netipsec/xform_ipip.c#10 (text+ko) ==== @@ -113,7 +113,7 @@ static int ipip_iattach(const void *); -VNET_MOD_DECLARE_STATELESS(IPIP, ipip, ipip_iattach, NULL) +VNET_MOD_DECLARE_STATELESS(IPIP, ipip, ipip_iattach, NULL, IPSEC) #ifdef INET6 /* ==== //depot/projects/vimage/src/sys/sys/vimage.h#29 (text+ko) ==== @@ -67,6 +67,7 @@ #define V_PROCG 2 #define V_CPU 3 +#define VNET_MOD_NONE -1 #define VNET_MOD_NET 0 #define VNET_MOD_NETGRAPH 1 #define VNET_MOD_INET 2 @@ -78,6 +79,8 @@ #define VNET_MOD_ALTQ 8 #define VNET_MOD_IPX 9 #define VNET_MOD_ATALK 10 +#define VNET_MOD_NG_IFACE 21 +#define VNET_MOD_NG_EIFACE 22 #define VNET_MOD_ESP 23 #define VNET_MOD_IPIP 24 #define VNET_MOD_AH 25 @@ -121,6 +124,7 @@ struct vnet_modinfo { int vmi_id; + int vmi_dependson; char *vmi_name; vnet_attach_fn *vmi_iattach; vnet_detach_fn *vmi_idetach; @@ -135,9 +139,11 @@ const char *vml_iname; }; -#define VNET_MOD_DECLARE(m_name_uc, m_name_lc, m_symmap, m_iattach, m_idetach) \ +#define VNET_MOD_DECLARE(m_name_uc, m_name_lc, m_iattach, m_idetach, \ + m_dependson, m_symmap) \ static const struct vnet_modinfo vnet_##m_name_lc##_modinfo = { \ .vmi_id = VNET_MOD_##m_name_uc, \ + .vmi_dependson = VNET_MOD_##m_dependson, \ .vmi_name = #m_name_lc, \ .vmi_iattach = m_iattach, \ .vmi_idetach = m_idetach, \ @@ -146,9 +152,11 @@ .vmi_symmap = m_symmap \ }; -#define VNET_MOD_DECLARE_STATELESS(m_name_uc, m_name_lc, m_iattach, m_idetach) \ +#define VNET_MOD_DECLARE_STATELESS(m_name_uc, m_name_lc, m_iattach, m_idetach, \ + m_dependson) \ static const struct vnet_modinfo vnet_##m_name_lc##_modinfo = { \ .vmi_id = VNET_MOD_##m_name_uc, \ + .vmi_dependson = VNET_MOD_##m_dependson, \ .vmi_name = #m_name_lc, \ .vmi_iattach = m_iattach, \ .vmi_idetach = m_idetach \