Date: Tue, 18 Sep 2018 07:53:33 +0900 From: KIRIYAMA Kazuhiko <kiri@kx.openedu.org> To: freebsd-net@freebsd.org Cc: kiri@kx.openedu.org Subject: NFS poor performance in ipfw_nat Message-ID: <201809172253.w8HMrXSS025987@kx.openedu.org>
next in thread | raw e-mail | index | archive | help
Hi, all
I'm working on ipfw_nat box with port redirect for sunrpc
(111) and nfsd (2049):
# uname -a
FreeBSD kx.xxx.org 12.0-CURRENT FreeBSD 12.0-CURRENT #0 r327038M: Fri Jan 5 16:16:33 JST 2018 admin@kx.xxx.org:/usr/obj/usr/src/amd64.amd64/sys/XIJ amd64
# cat /etc/rc.conf
defaultrouter="202.xxx.xxx.30"
hostname="kx.xxx.org"
ifconfig_em0="inet 202.xxx.xxx.26 netmask 255.255.255.248"
ifconfig_igb0="inet 192.168.1.254 netmask 255.255.255.0"
gateway_enable="YES"
firewall_enable="YES"
firewall_type="OPEN"
firewall_nat_enable="YES"
firewall_nat_interface="em0"
firewall_nat_flags="deny_in reset same_ports unreg_only"
firewall_nat_rules="/etc/ipfw_nat.rules"
nullfs_enable="YES"
zfs_enable="YES"
saver="logo"
sshd_enable="YES"
keymap=hy.armscii-8.kbd
ntpd_enable="YES"
ezjail_enable="YES"
sendmail_enable="NONE"
linux_enable="YES"
linux_adobe_enable="YES"
moused_nondefault_enable="NO"
#rc_debug="YES"
nfs_client_enable="YES"
#nfs_access_cache="8"
#nfs_bufpackets="64"
rpc_lockd_enable="YES"
rpc_statd_enable="YES"
# ifconfig
em0: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 1500
options=85259b<RXCSUM,TXCSUM,VLAN_MTU,VLAN_HWTAGGING,VLAN_HWCSUM,TSO4,LRO,WOL_MAGIC,VLAN_HWFILTER,VLAN_HWTSO>
ether 00:25:90:47:8d:c9
inet 202.xxx.xxx.26 netmask 0xfffffff8 broadcast 202.xxx.xxx.31
nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
media: Ethernet autoselect (1000baseT <full-duplex>)
status: active
igb0: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 1500
options=e505bb<RXCSUM,TXCSUM,VLAN_MTU,VLAN_HWTAGGING,JUMBO_MTU,VLAN_HWCSUM,TSO4,LRO,VLAN_HWFILTER,VLAN_HWTSO,RXCSUM_IPV6,TXCSUM_IPV6>
ether 00:25:90:47:8d:c8
inet 192.168.1.254 netmask 0xffffff00 broadcast 192.168.1.255
inet 192.168.1.2 netmask 0xffffffff broadcast 192.168.1.2
inet 192.168.1.4 netmask 0xffffffff broadcast 192.168.1.4
inet 192.168.1.3 netmask 0xffffffff broadcast 192.168.1.3
inet 192.168.1.5 netmask 0xffffffff broadcast 192.168.1.5
inet 192.168.1.1 netmask 0xffffffff broadcast 192.168.1.1
nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
media: Ethernet autoselect (1000baseT <full-duplex>)
status: active
lo0: flags=8049<UP,LOOPBACK,RUNNING,MULTICAST> metric 0 mtu 16384
options=600003<RXCSUM,TXCSUM,RXCSUM_IPV6,TXCSUM_IPV6>
inet6 ::1 prefixlen 128
inet6 fe80::1%lo0 prefixlen 64 scopeid 0x3
inet 127.0.0.1 netmask 0xff000000
nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
groups: lo
# ipfw list
00050 nat 123 ip4 from any to any via em0
00100 allow ip from any to any via lo0
00200 deny ip from any to 127.0.0.0/8
00300 deny ip from 127.0.0.0/8 to any
00400 deny ip from any to ::1
00500 deny ip from ::1 to any
00600 allow ipv6-icmp from :: to ff02::/16
00700 allow ipv6-icmp from fe80::/10 to fe80::/10
00800 allow ipv6-icmp from fe80::/10 to ff02::/16
00900 allow ipv6-icmp from any to any ip6 icmp6types 1
01000 allow ipv6-icmp from any to any ip6 icmp6types 2,135,136
65000 allow ip from any to any
65535 deny ip from any to any
# ipfw nat show config
ipfw nat 123 config if em0 log deny_in same_ports unreg_only reset redirect_port tcp 192.168.1.253:22 22253 redirect_port tcp 192.168.1.252:22 22252 redirect_port tcp 192.168.1.251:22 22251 redirect_port tcp 192.168.1.250:22 22250 redirect_port tcp 192.168.1.249:22 22249 redirect_port tcp 192.168.1.248:22 22248 redirect_port tcp 192.168.1.247:22 22247 redirect_port tcp 192.168.1.246:22 22246 redirect_port tcp 192.168.1.245:22 22245 redirect_port tcp 192.168.1.244:22 22244 redirect_port tcp 192.168.1.243:22 22243 redirect_port tcp 192.168.1.242:22 22242 redirect_port tcp 192.168.1.241:22 22241 redirect_port tcp 192.168.1.240:22 22240 redirect_port tcp 192.168.1.239:22 22239 redirect_port tcp 192.168.1.238:22 22238 redirect_port tcp 192.168.1.237:22 22237 redirect_port tcp 192.168.1.236:22 22236 redirect_port tcp 192.168.1.235:22 22235 redirect_port tcp 192.168.1.234:22 22234 redirect_port tcp 192.168.1.233:22 22233 redirect_port tcp 192.168.1.232:22 22232 redirect_port tcp 19!
2.168.1.231:22 22231 redirect_port tcp 192.168.1.230:22 22230 redirect_port tcp 192.168.1.229:22 22229 redirect_port tcp 192.168.1.228:22 22228 redirect_port tcp 192.168.1.227:22 22227 redirect_port tcp 192.168.1.226:22 22226 redirect_port tcp 192.168.1.225:22 22225 redirect_port tcp 192.168.1.224:22 22224 redirect_port tcp 192.168.1.223:22 22223 redirect_port tcp 192.168.1.222:22 22222 redirect_port tcp 192.168.1.221:22 22221 redirect_port tcp 192.168.1.220:22 22220 redirect_port tcp 192.168.1.219:22 22219 redirect_port tcp 192.168.1.218:22 22218 redirect_port tcp 192.168.1.217:22 22217 redirect_port tcp 192.168.1.216:22 22216 redirect_port tcp 192.168.1.215:22 22215 redirect_port tcp 192.168.1.214:22 22214 redirect_port tcp 192.168.1.213:22 22213 redirect_port tcp 192.168.1.212:22 22212 redirect_port tcp 192.168.1.211:22 22211 redirect_port tcp 192.168.1.210:22 22210 redirect_port tcp 192.168.1.209:22 22209 redirect_port tcp 192.168.1.208:22 22208 redirect_port tcp 192.16!
8.1.207:22 22207 redirect_port tcp 192.168.1.206:22 22206 redirect_port tcp 192.168.1.205:22 22205 redirect_port tcp 192.168.1.204:22 22204 redirect_port tcp 192.168.1.203:22 22203 redirect_port tcp 192.168.1.202:22 22202 redirect_port tcp 192.168.1.201:22 22201 redirect_port tcp 192.168.1.200:22 22200 redirect_port tcp 192.168.1.199:22 22199 redirect_port tcp 192.168.1.198:22 22198 redirect_port tcp 192.168.1.197:22 22197 redirect_port tcp 192.168.1.196:22 22196 redirect_port tcp 192.168.1.134:22 22134 redirect_port tcp 192.168.1.132:22 22132 redirect_port tcp 192.168.1.17:41920 22419 redirect_port tcp 192.168.1.5:22 22401 redirect_port tcp 192.168.1.4:22 22080 redirect_port tcp 192.168.1.3:22 22025 redirect_port tcp 192.168.1.2:22 22053 redirect_port tcp 192.168.1.1:22 22932 redirect_port tcp 192.168.1.254:41932 41932 redirect_port udp 192.168.1.254:2049 2049 redirect_port tcp 192.168.1.254:2049 2049 redirect_port udp 192.168.1.254:111 111 redirect_port tcp 192.168.1.254:111 111 r
edirect_port tcp 192.168.1.5:21 21 redirect_port tcp 192.168.1!
.4:80 80 redirect_port tcp 192.168.1.3:25 25
So, NFS mount vm.xxx.org:/.dake to /.dake (kx.xxx.org):
# df -htnfs
Filesystem Size Used Avail Capacity Mounted on
vm.xxx.org:/.dake 14T 48G 14T 0% /.dake
#
Then write files to vm.xxx.org:/.dake, but very slow in
transfer rate:
# dd if=/dev/zero of=/.dake/tmp/foo.img bs=1k count=1k
1024+0 records in
1024+0 records out
1048576 bytes transferred in 140.919168 secs (7441 bytes/sec)
#
This is done with noasync, but with async there is no distict diffrence:
# dd if=/dev/zero of=/.dake/tmp/foo.img bs=1k count=1k
1024+0 records in
1024+0 records out
1048576 bytes transferred in 141.384856 secs (7416 bytes/sec)
#
NFS server configuration is as follows:
# cat /etc/rc.conf
zfs_enable="YES"
defaultrouter="202.xxx.xxx.30"
ifconfig_igb0="inet 202.xxx.xxx.28 netmask 255.255.255.248"
ifconfig_igb1="inet 192.168.1.17 netmask 255.255.255.0"
hostname="vm.xxx.org"
mountd_enable="YES"
nfs_server_enable="YES"
nfs_server_flags="-h 192.168.1.17 -h 202.xxx.xxx.28 -u -t -n 6"
rpc_lockd_enable="YES"
rpc_statd_enable="YES"
sshd_enable="YES"
moused_enable="YES"
ntpd_enable="YES"
dumpdev="AUTO"
zfs_enable="YES"
sendmail_enable="NONE"
vm_enable="YES"
vm_dir="zfs:zroot/vm"
#
Is there any suggestions ?
Best regards
---
KIRIYAMA Kazuhiko
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201809172253.w8HMrXSS025987>