Date: Fri, 27 Sep 2019 16:31:41 +0300 From: "Alexander N. Lunev" <lan@zato.ru> To: freebsd-net@freebsd.org Subject: no network between jails and host with VNET on same interface Message-ID: <213f9284-5ddd-4dbc-6631-f8592efa2995@zato.ru>
next in thread | raw e-mail | index | archive | help
Hello everyone!
I have a strange connectivity problem on jails with VNET networking.
I've deployed a jail system with VNET networking on a server with
FreeBSD 12.0-RELEASE-p10. Jails are working fine, can reach out outer
network and each other, but there's no connectivity between host and jails.
Server is connected to switch trunk port by igb1 interface, which is
bridged with epairXa interfaces in bridge0, while jails using epairXb
interfaces (they are renamed to jail0 in each jail to simplify things).
======= host =============================
[igb1]-----------------------\
| +---------+
[vlan4 (10.1.1.247)] | |
| bridge0 |
/--[epair1a]----------------| |
/ +---------+
| /-[epair0a]--------------------/
| |
===== jail1_filter2 ======================
| \-[jail0(ex-epair0b)]
| |
| [vlan4 (10.1.1.26)]
===== jail2_noc ==========================
\-[jail0(ex-epair1b)]
|
[vlan4 (10.1.1.201)]
===========================================
On the host and in every jail i have a vlan4 interface, and here's
addresses for those vlan4 interfaces:
host@vlan4: 10.1.1.247
jail1_filter2@vlan4: 10.1.1.26
jail2_noc@vlan4: 10.1.1.201
Host can't ping jails, but can ping outer world. Jails can ping each
other and outer world, but not host - "ping: sendto: Host is down",
there's no ARP entry for host' vlan4 address.
I've tried to add static arp entry for 10.1.1.247 in jails - with no
success (arp is added, network still not working).
Host and both jails have firewall_type=OPEN configured.
What is wrong here?
Below are pings, ifconfig output for host and jails and jail.conf.
From jail1_filter2:
root@jail1_filter2:/ # ping 10.1.1.201
PING 10.1.1.201 (10.1.1.201): 56 data bytes
64 bytes from 10.1.1.201: icmp_seq=0 ttl=64 time=0.688 ms
64 bytes from 10.1.1.201: icmp_seq=1 ttl=64 time=0.552 ms
root@jail1_filter2:/ # ping 10.1.1.247
PING 10.1.1.247 (10.1.1.247): 56 data bytes
ping: sendto: Host is down
ping: sendto: Host is down
root@jail1_filter2:/ # arp 10.1.1.247
ibm.zato.local (10.1.1.247) at (incomplete) on vlan4 expired [vlan]
From jail2_noc:
root@jail2_noc:/ # ping 10.1.1.26
PING 10.1.1.26 (10.1.1.26): 56 data bytes
64 bytes from 10.1.1.26: icmp_seq=0 ttl=64 time=0.542 ms
64 bytes from 10.1.1.26: icmp_seq=1 ttl=64 time=0.510 ms
root@jail2_noc:/ # ping 10.1.1.247
PING 10.1.1.247 (10.1.1.247): 56 data bytes
ping: sendto: Host is down
ping: sendto: Host is down
root@jail2_noc:/ # arp 10.1.1.247
ibm.zato.local (10.1.1.247) at (incomplete) on vlan4 expired [vlan]
Jail's ifconfig:
root@jail2_noc:/ # ifconfig
jail0: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 1500
options=8<VLAN_MTU>
ether 02:3e:b2:17:83:0b
groups: epair
media: Ethernet 10Gbase-T (10Gbase-T <full-duplex>)
status: active
nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
vlan4: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 1500
ether 02:3e:b2:17:83:0b
inet 10.1.1.201 netmask 0xffffff00 broadcast 10.1.1.255
groups: vlan
vlan: 4 vlanpcp: 0 parent interface: jail0
media: Ethernet 10Gbase-T (10Gbase-T <full-duplex>)
status: active
nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
root@jail1_filter2:/ # ifconfig
jail0: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 1500
options=8<VLAN_MTU>
ether 02:09:d8:ee:38:0b
inet 178.255.248.13 netmask 0xffffffe0 broadcast 178.255.248.31
groups: epair
media: Ethernet 10Gbase-T (10Gbase-T <full-duplex>)
status: active
nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
vlan4: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 1500
ether 02:09:d8:ee:38:0b
inet 10.1.1.26 netmask 0xffffff00 broadcast 10.1.1.255
inet6 fe80::9:d8ff:feee:380b%vlan4 prefixlen 64 scopeid 0x3
groups: vlan
vlan: 4 vlanpcp: 0 parent interface: jail0
media: Ethernet 10Gbase-T (10Gbase-T <full-duplex>)
status: active
nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
Host's ifconfig:
igb1: flags=8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST> metric
0 mtu 1500
options=a520b9<RXCSUM,VLAN_MTU,VLAN_HWTAGGING,JUMBO_MTU,VLAN_HWCSUM,WOL_MAGIC,VLAN_HWFILTER,VLAN_HWTSO,RXCSUM_IPV6>
ether 40:f2:e9:0e:8c:65
media: Ethernet autoselect (1000baseT <full-duplex>)
status: active
nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
vlan4: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 1500
options=200001<RXCSUM,RXCSUM_IPV6>
ether 40:f2:e9:0e:8c:65
inet 10.1.1.247 netmask 0xffffff00 broadcast 10.1.1.255
groups: vlan
vlan: 4 vlanpcp: 0 parent interface: igb1
media: Ethernet autoselect (1000baseT <full-duplex>)
status: active
nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
bridge0: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu
1500
ether b6:cb:19:20:0f:60
id 00:00:00:00:00:00 priority 32768 hellotime 2 fwddelay 15
maxage 20 holdcnt 6 proto rstp maxaddr 2000 timeout 1200
root id 00:00:00:00:00:00 priority 32768 ifcost 0 port 0
member: epair1a flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
ifmaxaddr 0 port 10 priority 128 path cost 2000
member: epair0a flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
ifmaxaddr 0 port 9 priority 128 path cost 2000
member: igb1 flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
ifmaxaddr 0 port 2 priority 128 path cost 20000
groups: bridge
nd6 options=1<PERFORMNUD>
epair0a: flags=8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST>
metric 0 mtu 1500
options=8<VLAN_MTU>
ether 02:09:d8:ee:38:0a
inet6 fe80::9:d8ff:feee:380a%epair0a prefixlen 64 scopeid 0x9
groups: epair
media: Ethernet 10Gbase-T (10Gbase-T <full-duplex>)
status: active
nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
epair1a: flags=8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST>
metric 0 mtu 1500
options=8<VLAN_MTU>
ether 02:3e:b2:17:83:0a
inet6 fe80::3e:b2ff:fe17:830a%epair1a prefixlen 64 scopeid 0xa
groups: epair
media: Ethernet 10Gbase-T (10Gbase-T <full-duplex>)
status: active
nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
jail.conf:
========================
$uplinkdev = "igb1";
$jbridge = "bridge0";
$vn_iface = "jail0";
vnet.interface = "$vn_iface";
vnet;
exec.prestart = "ifconfig $jbridge > /dev/null 2> /dev/null || (
ifconfig $jbridge create up && ifconfig $jbridge addm $uplinkdev )";
exec.prestart += "ifconfig $epair create up || echo
'Skipped creating epair (exists?)'";
exec.prestart += "ifconfig $jbridge addm ${epair}a || echo
'Skipped adding bridge member (already member?)'";
exec.created = "ifconfig ${epair}b name $vn_iface ||
echo 'Skipped renaming ifdev to $vn_iface (looks bad...)'";
exec.poststop = "ifconfig $jbridge deletem ${epair}a";
exec.poststop += "ifconfig ${epair}a destroy";
filter2 {
$epair = "epair0";
}
noc {
$epair = "epair1";
}
===========================
--
Best regards,
Alexander Lunev
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?213f9284-5ddd-4dbc-6631-f8592efa2995>