Date: Thu, 23 May 2013 18:12:07 -0400 From: Joe <fbsd8@a1poweruser.com> To: freebsd-questions@Freebsd.org Subject: netgraph network for jail(8) vnet jail unable to reach internet Message-ID: <519E9437.6010100@a1poweruser.com>
next in thread | raw e-mail | index | archive | help
Hello list.
Trying to get my script to work that creates a netgraph network for a
jail(8) vnet jail. Every thing seems to work, but from inside of the
started vnet jail I can not ping the public internet. The host can ping
the public internet so the problem has to be in the netgraph script. The
problem must be staring me in the face but I can just not see it. A
fresh pair of eyes may see things I am missing.
I'm running 9.1-RELEASE with vimage compiled into the kernel. Non-vnet
jails work fine and bridge/epair networked vnet jails work fine. The
host has a single ethernet interface (rl0) facing the public internet.
Dhcp is used to get the hosts ip address and dns server info. The
vnet.ng script is designed to create a single ng bridge to rl0 and
connect vnet jails to it as the jails are started. The following is a
walk through of a test cycle showing what I can see from the host. At
the end is a listing of the vnet.ng script.
Thanks for your help
# From the host lets see if there is a netgraph network before we start?
# Nope, no netgraph network running
# /root >ngctl ls -l
There are 2 total nodes:
Name: rl0 Type: ether ID: 00000001 Num hooks: 0
Name: ngctl2850 Type: socket ID: 00000037 Num hooks: 0
# Here is the jail(8) jail.conf definition statements
# /root >cat /usr/local/etc/vnet/vdir9
vdir9 {
host.hostname = "vdir9";
path = "/usr/jails/vdir9";
mount.fstab = "/usr/local/etc/fstab/vdir9";
exec.start = "/bin/sh /etc/rc";
exec.stop = "/bin/sh /etc/rc.shutdown";
exec.consolelog = "/var/log/vdir9.console.log";
devfs_ruleset = "4";
allow.mount.devfs;
vnet;
}
# Lets start the vnet jail
# /root >jail -f /usr/local/etc/qjail.vnet/vdir9 -c
vdir9: created
# Yes it’s really running.
# /root >jls
JID IP Address Hostname Path
4 - vdir9 /usr/jails/vdir9
# Lets start the netgraph network for the running vnet jail
# /root >vnet.ng start vdir9 rl0
Netgraph vnet jail network established successfully!
# Lets check the host for the vnet jail netgraph network
# Yep it’s there and looks complete to me.
# /root >ngctl ls -l
There are 4 total nodes:
Name: rl0 Type: ether ID: 00000001 Num hooks: 2
Local hook Peer name Peer type Peer ID Peer
hook
---------- --------- --------- -------
---------
upper bridge0 bridge 0000003d link1
lower bridge0 bridge 0000003d link0
Name: bridge0 Type: bridge ID: 0000003d Num hooks: 3
Local hook Peer name Peer type Peer ID Peer
hook
---------- --------- --------- -------
---------
link2 vdir9 eiface 00000041 ether
link1 rl0 ether 00000001 upper
link0 rl0 ether 00000001 lower
Name: vdir9 Type: eiface ID: 00000041 Num hooks: 1
Local hook Peer name Peer type Peer ID Peer
hook
---------- --------- --------- -------
---------
ether bridge0 bridge 0000003d link2
Name: ngctl3126 Type: socket ID: 00000046 Num hooks: 0
# Lets log into the running jail
# /root >jexec vdir9 tcsh
# Lets ping freebsd.org ip address. No public internet connection
# even though host can do same ping and get good reply.
vdir9 / >ping -c4 8.8.178.135
PING 8.8.178.135 (8.8.178.135): 56 data bytes
--- 8.8.178.135 ping statistics ---
4 packets transmitted, 0 packets received, 100.0% packet loss
vdir9 / >exit
exit
# Now stop the vnet jails netgraph network
# /root >vnet.ng stop vdir9 rl0
Netgraph vnet jail network shutdown successfully!
# and yes the netgraph network is shutdown
# /root >ngctl ls -l
There are 2 total nodes:
Name: rl0 Type: ether ID: 00000001 Num hooks: 0
Name: ngctl3167 Type: socket ID: 0000004b Num hooks: 0
# Here is the vnet.ng script
# It starts and stops the vnet jail’s netgraph network
#!/bin/sh
function=$1
jailname=$2
nicname=$3
jid=`jls -j ${jailname} jid`
# Load netgraph kernel modules if not done already.
for module in ng_socket netgraph ng_bridge ng_eiface ng_ether; do
if ! kldstat -v | grep -qw ${module}; then
kldload ${module} || exit 1
fi
done
# mac manufacturer prefix. Modify if need be.
#mac_prefix="00:1d:92"
mac_prefix="07:22:49"
start() {
sysctl net.inet.ip.forwarding=1 > /dev/null 2> /dev/null
jid=`jls -j ${jailname} jid`
if [ "${jid}" -gt "100" ]; then
echo " "
echo "WARNING: The JID value is greater then 100."
echo "This may indicate many cycles of starting/stopping vnet jails"
echo "which results in lost memory pages. To recover the lost memory,"
echo "shutdown the host and reboot. This will zero out the JID"
echo "counter and make all the memory available again."
echo " "
fi
# The jid is used in the jails ip address which has max value of 255,
# so draw the line at 250 running vnet jails and then force reboot.
#
if [ "${jid}" -gt "250" ]; then
echo " "
echo "ERROR: No more vnet jail ip addresses can be created."
echo "You MUST shutdown the host and reboot before vnet jails are"
echo "startable again."
echo " "
exit 2
fi
# Check real ethernet interface exists.
#
if ! ngctl info ${nicname}: >/dev/null 2>&1; then
echo "Error: interface ${nicname} does not exist."
exit 1
fi
ifconfig ${nicname} up || exit 1
# Check if this ethernet interface has a bridge already created on it.
# If not then create bridge to the ethernet interface.
#
bridge_name=`ngctl show ${nicname}: | grep -m 1 bridge | awk '{print $2}'`
if [ -z "${bridge_name}" ]; then
# This means no bridge created yet on this ethernet interface.
# Set NIC interfaces to promiscuous mode and don't overwrite src addr.
ngctl msg ${nicname}: setpromisc 1 || exit 1
ngctl msg ${nicname}: setautosrc 0 || exit 1
bridge_name="bridge0"
# Create new ng_bridge node and attach it to the ethernet interface,
# connecting ng_ether:lower hook to bridge:link0
#
ngctl mkpeer ${nicname}: bridge lower link0 || exit 1
# Assigning ${bridge_name} as bridge name.
ngctl name ${nicname}:lower ${bridge_name} || exit 1
# Connect ng_ether:upper hook to bridge:link1
# after bridge name is set.
#
ngctl connect ${nicname}: ${bridge_name}: upper link1 || exit 1
fi
# Create and link vnet jail to bridge.
#
ngctl mkpeer eiface ether ether || exit 1
org_eiface=`ngctl l | grep ngeth | tail -n 1 | awk '{print $2}'`
# Renaming vnet jail ng node interface name to jailname.
ngctl name ${org_eiface}: ${jailname} || exit 1
eiface="${jailname}"
ifconfig ${org_eiface} name ${eiface}
# Finding next link number for this bridge.
used_linknum=`ngctl show bridge0: | grep link | cut -c 7-7`
highest_linknum=0
for used_num in ${used_linknum}; do
if [ "${used_num}" -gt "${highest_linknum}" ]; then
highest_linknum="${used_num}"
fi
done
linknum=`expr ${highest_linknum} + 1`
# Connect vnet jail virtual interface to bridge interface.
#
ngctl connect ${eiface}: ${bridge_name}: ether link${linknum} || exit 1
# Move host virtual interface to vnet virtual node.
# Moved interface can be seen in the output of ifconfig command issued
# from within the started vnet jail or by issuing
# 'jexec jailname ifconfig' from the host system.
#
ifconfig ${eiface} vnet ${jailname} || exit 1
# Make lo0 interface localhost.
jexec ${jailname} ifconfig lo0 localhost || exit 1
# Set the MAC address of the new interface using a sensible
# algorithm to prevent conflicts on the network. First three
# octets can be changed by user. Last three octets are
# generated randomly.
#
M4=`od -An -N2 -i /dev/random | sed -e 's/ //g' | \
awk '{ print $1 % 256 }'`
M5=`od -An -N2 -i /dev/random | sed -e 's/ //g' | \
awk '{ print $1 % 256 }'`
M6=`od -An -N2 -i /dev/random | sed -e 's/ //g' | \
awk '{ print $1 % 256 }'`
mac=`printf ${mac_prefix}:%02x:%02x:%02x ${M4} ${M5} ${M6}`
# Setting mac address of ${eiface} to ${mac}
jexec ${jailname} ifconfig ${eiface} link $mac || exit 1
# Set IP address inside of vnet jail ng virtual interface.
jexec ${jailname} ifconfig ${eiface} inet 10.${jid}.0.2 || exit 1
# Assign default route ip address to ng virtual interface
# inside of the vnet jail.
#
jexec ${jailname} route add default 10.${jid}.0.1 || exit 1
echo "Netgraph vnet jail network established successfully!"
}
stop() {
ngctl shutdown ${jailname}: >/dev/null 2>&1
# Get the bridge name for this NIC.
#
bridge_name=`ngctl show ${nicname}: | grep -m 1 bridge | awk '{print $2}'`
# If there are only 2 links (link0 & link1) on the bridge name,
# this means all the vnet jails have been shutdown so shutdown the bridge.
#
linknum=`ngctl show ${bridge_name}: | grep link | wc -l | sed -e "s/ //g"`
[ ${linknum} -eq 2 ] && ngctl shutdown ${bridge_name}: >/dev/null 2>&1
echo "Netgraph vnet jail network shutdown successfully!"
}
# Main entry point.
#
[ "${function}" = "start" ] && start && exit 0
[ "${function}" = "stop" ] && stop && exit 0
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?519E9437.6010100>