Date: Mon, 09 May 2011 15:48:25 +0300 From: Mikolaj Golub <trociny@freebsd.org> To: freebsd-virtualization@FreeBSD.org Cc: Kostik Belousov <kib@FreeBSD.org>, Pawel Jakub Dawidek <pjd@FreeBSD.org> Subject: vnet: acessing module's virtualized global variables from another module Message-ID: <86aaewdopy.fsf@kopusha.home.net>
next in thread | raw e-mail | index | archive | help
--=-=-=
Hi,
Trying ipfw_nat under VIMAGE kernel I got this panic on the module load:
Fatal trap 12: page fault while in kernel mode
cpuid = 1; apic id = 01
fault virtual address = 0x4
fault code = supervisor read, page not present
instruction pointer = 0x20:0xc09f098e
stack pointer = 0x28:0xf563b944
frame pointer = 0x28:0xf563b998
code segment = base 0x0, limit 0xfffff, type 0x1b
= DPL 0, pres 1, def32 1, gran 1
processor eflags = interrupt enabled, resume, IOPL = 0
current process = 4264 (kldload)
witness_checkorder(c6d5e91c,9,ca0ac2e3,223,0,...) at witness_checkorder+0x6e
_rw_wlock(c6d5e91c,ca0ac2e3,223,0,c0e8f795,...) at _rw_wlock+0x82
ipfw_nat_modevent(c98a48c0,0,0,75,0,...) at ipfw_nat_modevent+0x41
module_register_init(ca0ad508,0,c0e8d834,e6,0,...) at module_register_init+0xa7
linker_load_module(0,f563bc18,c0e8d834,3fc,f563bc28,...) at linker_load_module+0xa05
kern_kldload(c86835c0,c72d3400,f563bc40,0,c8d0d000,...) at kern_kldload+0x133
kldload(c86835c0,f563bcec,c09e8940,c86835c0,0,...) at kldload+0x74
syscallenter(c86835c0,f563bce4,c0ce05dd,c1022150,0,...) at syscallenter+0x263
syscall(f563bd28) at syscall+0x34
Xint0x80_syscall() at Xint0x80_syscall+0x21
--- syscall (304, FreeBSD ELF32, kldload), eip = 0x280da00b, esp = 0xbfbfe79c, ebp = 0xbfbfec88 -
It crashed on acessing data from virtualized global variable V_layer3_chain in
ipfw_nat_modevent(). V_layer3_chain is defined in ipfw module and it turns out
that &V_layer3_chain returns wrong location from anywhere but ipfw.ko.
May be this is a known issue, but I have not found info about this, so below
are details of investigation why this happens.
Virtualized global variables are defined using the VNET_DEFINE() macro, which
places them in the 'set_vnet' linker set (in the base kernel or in
module). This is used to
1) copy these "default" values to each virtual network stack instance when created;
2) act as unique global names by which the variable can be referred to. The
location of a per-virtual instance variable is calculated at run-time like in
the example below for layer3_chain variable in the default vnet (vnet0):
vnet0->vnet_data_base + (uintptr_t) & vnet_entry_layer3_chain (1)
For modules the thing is more complicated. When a module is loaded its global
variables from 'set_vnet' linker set are copied to the kernel 'set_vnet', and
for module to be able to access them the linker reallocates all references
accordingly (kern/link_elf.c:elf_relocaddr()):
if (x >= ef->vnet_start && x < ef->vnet_stop)
return ((x - ef->vnet_start) + ef->vnet_base);
So from inside the module the access to its virtualized variables works, but
from the outside we get wrong location using calculation like above (1),
because &vnet_entry_layer3_chain returns address of the variable in the
module's 'set_vnet'.
The workaround is to compile such modules into the kernel or use a hack I have
done for ipfw_nat -- add the function to ipfw module which returns the
location of virtualized layer3_chain variable and use this location instead of
V_layer3_chain macro (see the attached patch).
But I suppose the problem is not a new and there might be better approach
already invented to deal with this?
--
Mikolaj Golub
--=-=-=
Content-Type: text/x-patch
Content-Disposition: attachment; filename=ipfw_nat.patch
Index: sys/netinet/ipfw/ip_fw_private.h
===================================================================
--- sys/netinet/ipfw/ip_fw_private.h (revision 221673)
+++ sys/netinet/ipfw/ip_fw_private.h (working copy)
@@ -201,6 +201,8 @@ VNET_DECLARE(int, fw_verbose);
VNET_DECLARE(struct ip_fw_chain, layer3_chain);
#define V_layer3_chain VNET(layer3_chain)
+void* vnet_entry_addr_layer3_chain(void);
+#define V_addr_layer3_chain (vnet_entry_addr_layer3_chain())
VNET_DECLARE(u_int32_t, set_disable);
#define V_set_disable VNET(set_disable)
Index: sys/netinet/ipfw/ip_fw_nat.c
===================================================================
--- sys/netinet/ipfw/ip_fw_nat.c (revision 221673)
+++ sys/netinet/ipfw/ip_fw_nat.c (working copy)
@@ -62,7 +62,7 @@ ifaddr_change(void *arg __unused, struct ifnet *if
struct ifaddr *ifa;
struct ip_fw_chain *chain;
- chain = &V_layer3_chain;
+ chain = V_addr_layer3_chain;
IPFW_WLOCK(chain);
/* Check every nat entry... */
LIST_FOREACH(ptr, &chain->nat, _next) {
@@ -345,7 +345,7 @@ ipfw_nat_cfg(struct sockopt *sopt)
{
struct cfg_nat *cfg, *ptr;
char *buf;
- struct ip_fw_chain *chain = &V_layer3_chain;
+ struct ip_fw_chain *chain = V_addr_layer3_chain;
size_t len;
int gencnt, error = 0;
@@ -421,7 +421,7 @@ static int
ipfw_nat_del(struct sockopt *sopt)
{
struct cfg_nat *ptr;
- struct ip_fw_chain *chain = &V_layer3_chain;
+ struct ip_fw_chain *chain = V_addr_layer3_chain;
int i;
sooptcopyin(sopt, &i, sizeof i, sizeof i);
@@ -444,7 +444,7 @@ ipfw_nat_del(struct sockopt *sopt)
static int
ipfw_nat_get_cfg(struct sockopt *sopt)
{
- struct ip_fw_chain *chain = &V_layer3_chain;
+ struct ip_fw_chain *chain = V_addr_layer3_chain;
struct cfg_nat *n;
struct cfg_redir *r;
struct cfg_spool *s;
@@ -509,7 +509,7 @@ ipfw_nat_get_log(struct sockopt *sopt)
int i, size;
struct ip_fw_chain *chain;
- chain = &V_layer3_chain;
+ chain = V_addr_layer3_chain;
IPFW_RLOCK(chain);
/* one pass to count, one to copy the data */
@@ -543,8 +543,11 @@ ipfw_nat_get_log(struct sockopt *sopt)
static void
ipfw_nat_init(void)
{
+ struct ip_fw_chain *chain;
- IPFW_WLOCK(&V_layer3_chain);
+ chain = V_addr_layer3_chain;
+
+ IPFW_WLOCK(chain);
/* init ipfw hooks */
ipfw_nat_ptr = ipfw_nat;
lookup_nat_ptr = lookup_nat;
@@ -552,7 +555,7 @@ ipfw_nat_init(void)
ipfw_nat_del_ptr = ipfw_nat_del;
ipfw_nat_get_cfg_ptr = ipfw_nat_get_cfg;
ipfw_nat_get_log_ptr = ipfw_nat_get_log;
- IPFW_WUNLOCK(&V_layer3_chain);
+ IPFW_WUNLOCK(chain);
V_ifaddr_event_tag = EVENTHANDLER_REGISTER(
ifaddr_event, ifaddr_change,
NULL, EVENTHANDLER_PRI_ANY);
@@ -564,7 +567,7 @@ ipfw_nat_destroy(void)
struct cfg_nat *ptr, *ptr_temp;
struct ip_fw_chain *chain;
- chain = &V_layer3_chain;
+ chain = V_addr_layer3_chain;
IPFW_WLOCK(chain);
LIST_FOREACH_SAFE(ptr, &chain->nat, _next, ptr_temp) {
LIST_REMOVE(ptr, _next);
Index: sys/netinet/ipfw/ip_fw2.c
===================================================================
--- sys/netinet/ipfw/ip_fw2.c (revision 221673)
+++ sys/netinet/ipfw/ip_fw2.c (working copy)
@@ -134,6 +134,11 @@ VNET_DEFINE(int, verbose_limit);
/* layer3_chain contains the list of rules for layer 3 */
VNET_DEFINE(struct ip_fw_chain, layer3_chain);
+void*
+vnet_entry_addr_layer3_chain(void)
+{
+ return &V_layer3_chain;
+}
ipfw_nat_t *ipfw_nat_ptr = NULL;
struct cfg_nat *(*lookup_nat_ptr)(struct nat_list *, int);
--=-=-=--
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?86aaewdopy.fsf>