Skip site navigation (1)Skip section navigation (2)
Date:      Sat, 6 Aug 2022 16:53:33 -0400
From:      mike tancsa <mike@sentex.net>
To:        Jim King <jim@jimking.net>, stable@freebsd.org, kbowling@FreeBSD.org
Subject:   Re: igc problems with heavy traffic
Message-ID:  <760d007e-0a19-7f29-ade6-69a81744902d@sentex.net>
In-Reply-To: <a4ddc96a-3dd5-4fee-8003-05f228d10858@jimking.net>
References:  <fc256428-3ff1-68ba-cfcc-a00ca427e85b@jimking.net> <59b9cec0-d8c2-ce72-b5e9-99d1a1e807f8@sentex.net> <e714cd76-0aaa-3ea0-3c31-5e61badffa18@sentex.net> <86995d10-af63-d053-972e-dd233029f3bf@jimking.net> <3d874f65-8ce2-8f06-f19a-14cd550166e3@sentex.net> <a8192d60-2970-edb5-ce1a-c17ea875bf07@jimking.net> <fd1e825b-c306-64b1-f9ef-fec0344a9c95@sentex.net> <a4ddc96a-3dd5-4fee-8003-05f228d10858@jimking.net>

next in thread | previous in thread | raw e-mail | index | archive | help
On 8/6/2022 3:04 PM, Jim King wrote:
> On 8/5/2022 12:47 PM, mike tancsa wrote:
>> On 8/5/2022 11:41 AM, Jim King wrote:
>>> If I fiddle with the Flow Control settings it seems to make a 
>>> difference.  I can do it without bouncing the link.  I wonder what 
>>> the Linux driver has as default ?
>>>>
>>>> ie.
>>>>
>>>> sysctl -w dev.igc.1.fc=1
>>>> sysctl -w dev.igc.0.fc=1
>>>>
>>>> Does it help in your case ?
>>>>
>>> Thanks for the suggestion.  Unfortunately it doesn't seem to help in 
>>> my case.  All four values for flow control give pretty much the same 
>>> results with my iperf test.
>>>
>>>
>> Are you able to boot a RELENG_13 stable kernel and test ? I am using 
>> a stable kernel from a couple of days ago and setting the fc=0 (you 
>> set that on all nics right ?) does work for me both at 2.5G and 1G. 
>> Leaving the default causes the nic to bounce on my test boxes. I dont 
>> see any obvious big changes in the driver itself since 13.1 came out 
>> so I am guessing something else fixed or worked around this bug ?
>>
> I build stable/13 from yesterday and still haven't found a working 
> configuration.  When I turn off Flow Control on both NICs my first 
> iperf test always causes the NIC to bounce, and sometimes subsequent 
> tests will work OK, but only sometimes.  Most of the time the 
> subsequent iperf runs also cause the NIC to bounce.
>
> I also tried -rxcsum and -txcsum on the NIC, which had no effect.
>
>
I let my unit run over night and it as well saw some drops.  I wonder if 
the relatively underpowered CPU doesnt generate enough traffic to cause 
it to bounce as often?  Not sure if there is anything in the sysctl logs 
or not, but this is a before and after below. I guess I will open a PR. 
Not sure if this driver is managed by Intel or not. I think the netgate 
people did it initially

root@topton1:/var/tmp/reports # diff -u 1659776941b 1659777187a
--- 1659776941b 2022-08-06 05:11:03.514094000 -0400
+++ 1659777187a 2022-08-06 05:14:09.675013000 -0400
@@ -1,35 +1,35 @@
  dev.igc.1.wake: 0
  dev.igc.1.interrupts.rx_desc_min_thresh: 0
-dev.igc.1.interrupts.asserts: 704721082
-dev.igc.1.mac_stats.tso_txd: 400062204
-dev.igc.1.mac_stats.tx_frames_1024_1522: 4870151883
-dev.igc.1.mac_stats.tx_frames_512_1023: 38990
-dev.igc.1.mac_stats.tx_frames_256_511: 17758
-dev.igc.1.mac_stats.tx_frames_128_255: 4764
-dev.igc.1.mac_stats.tx_frames_65_127: 569597300
-dev.igc.1.mac_stats.tx_frames_64: 36196
+dev.igc.1.interrupts.asserts: 707056694
+dev.igc.1.mac_stats.tso_txd: 400573812
+dev.igc.1.mac_stats.tx_frames_1024_1522: 4882409702
+dev.igc.1.mac_stats.tx_frames_512_1023: 38993
+dev.igc.1.mac_stats.tx_frames_256_511: 17762
+dev.igc.1.mac_stats.tx_frames_128_255: 4769
+dev.igc.1.mac_stats.tx_frames_65_127: 572463728
+dev.igc.1.mac_stats.tx_frames_64: 36307
  dev.igc.1.mac_stats.mcast_pkts_txd: 0
  dev.igc.1.mac_stats.bcast_pkts_txd: 30
-dev.igc.1.mac_stats.good_pkts_txd: 5439846882
-dev.igc.1.mac_stats.total_pkts_txd: 5439846888
-dev.igc.1.mac_stats.good_octets_txd: 7432797311490
-dev.igc.1.mac_stats.good_octets_recvd: 7400832173605
-dev.igc.1.mac_stats.rx_frames_1024_1522: 4849042867
-dev.igc.1.mac_stats.rx_frames_512_1023: 62119
-dev.igc.1.mac_stats.rx_frames_256_511: 28123
-dev.igc.1.mac_stats.rx_frames_128_255: 5910
-dev.igc.1.mac_stats.rx_frames_65_127: 570440786
-dev.igc.1.mac_stats.rx_frames_64: 49169
+dev.igc.1.mac_stats.good_pkts_txd: 5454971260
+dev.igc.1.mac_stats.total_pkts_txd: 5454971261
+dev.igc.1.mac_stats.good_octets_txd: 7451605352901
+dev.igc.1.mac_stats.good_octets_recvd: 7435895205487
+dev.igc.1.mac_stats.rx_frames_1024_1522: 4872086268
+dev.igc.1.mac_stats.rx_frames_512_1023: 64952
+dev.igc.1.mac_stats.rx_frames_256_511: 29389
+dev.igc.1.mac_stats.rx_frames_128_255: 6311
+dev.igc.1.mac_stats.rx_frames_65_127: 571594440
+dev.igc.1.mac_stats.rx_frames_64: 49437
  dev.igc.1.mac_stats.mcast_pkts_recvd: 0
  dev.igc.1.mac_stats.bcast_pkts_recvd: 19
-dev.igc.1.mac_stats.good_pkts_recvd: 5419628981
-dev.igc.1.mac_stats.total_pkts_recvd: 5419629039
+dev.igc.1.mac_stats.good_pkts_recvd: 5443830806
+dev.igc.1.mac_stats.total_pkts_recvd: 5443830878
  dev.igc.1.mac_stats.xoff_txd: 0
  dev.igc.1.mac_stats.xoff_recvd: 14
  dev.igc.1.mac_stats.xon_txd: 0
  dev.igc.1.mac_stats.xon_recvd: 0
  dev.igc.1.mac_stats.alignment_errs: 0
-dev.igc.1.mac_stats.crc_errs: 6
+dev.igc.1.mac_stats.crc_errs: 8
  dev.igc.1.mac_stats.recv_errs: 0
  dev.igc.1.mac_stats.recv_jabber: 0
  dev.igc.1.mac_stats.recv_oversize: 0
@@ -40,43 +40,43 @@
  dev.igc.1.mac_stats.defer_count: 0
  dev.igc.1.mac_stats.sequence_errors: 0
  dev.igc.1.mac_stats.symbol_errors: 0
-dev.igc.1.mac_stats.collision_count: 6
+dev.igc.1.mac_stats.collision_count: 8
  dev.igc.1.mac_stats.late_coll: 0
  dev.igc.1.mac_stats.multiple_coll: 0
  dev.igc.1.mac_stats.single_coll: 0
  dev.igc.1.mac_stats.excess_coll: 0
  dev.igc.1.queue_rx_3.rx_irq: 0
-dev.igc.1.queue_rx_3.rxd_tail: 907
-dev.igc.1.queue_rx_3.rxd_head: 908
+dev.igc.1.queue_rx_3.rxd_tail: 128
+dev.igc.1.queue_rx_3.rxd_head: 0
  dev.igc.1.queue_rx_2.rx_irq: 0
-dev.igc.1.queue_rx_2.rxd_tail: 809
-dev.igc.1.queue_rx_2.rxd_head: 812
+dev.igc.1.queue_rx_2.rxd_tail: 722
+dev.igc.1.queue_rx_2.rxd_head: 723
  dev.igc.1.queue_rx_1.rx_irq: 0
-dev.igc.1.queue_rx_1.rxd_tail: 401
-dev.igc.1.queue_rx_1.rxd_head: 402
+dev.igc.1.queue_rx_1.rxd_tail: 83
+dev.igc.1.queue_rx_1.rxd_head: 84
  dev.igc.1.queue_rx_0.rx_irq: 0
-dev.igc.1.queue_rx_0.rxd_tail: 521
-dev.igc.1.queue_rx_0.rxd_head: 522
+dev.igc.1.queue_rx_0.rxd_tail: 228
+dev.igc.1.queue_rx_0.rxd_head: 229
  dev.igc.1.queue_tx_3.tx_irq: 0
-dev.igc.1.queue_tx_3.txd_tail: 1023
-dev.igc.1.queue_tx_3.txd_head: 1023
+dev.igc.1.queue_tx_3.txd_tail: 750
+dev.igc.1.queue_tx_3.txd_head: 750
  dev.igc.1.queue_tx_2.tx_irq: 0
-dev.igc.1.queue_tx_2.txd_tail: 770
-dev.igc.1.queue_tx_2.txd_head: 914
+dev.igc.1.queue_tx_2.txd_tail: 116
+dev.igc.1.queue_tx_2.txd_head: 116
  dev.igc.1.queue_tx_1.tx_irq: 0
-dev.igc.1.queue_tx_1.txd_tail: 184
-dev.igc.1.queue_tx_1.txd_head: 643
+dev.igc.1.queue_tx_1.txd_tail: 347
+dev.igc.1.queue_tx_1.txd_head: 347
  dev.igc.1.queue_tx_0.tx_irq: 0
-dev.igc.1.queue_tx_0.txd_tail: 210
-dev.igc.1.queue_tx_0.txd_head: 210
+dev.igc.1.queue_tx_0.txd_tail: 17
+dev.igc.1.queue_tx_0.txd_head: 17
  dev.igc.1.fc_low_water: 32752
  dev.igc.1.fc_high_water: 32768
  dev.igc.1.rx_control: 71335966
  dev.igc.1.device_control: 404489793
  dev.igc.1.watchdog_timeouts: 0
  dev.igc.1.rx_overruns: 0
-dev.igc.1.link_irq: 38
-dev.igc.1.dropped: 6
+dev.igc.1.link_irq: 42
+dev.igc.1.dropped: 8
  dev.igc.1.eee_control: 1
  dev.igc.1.itr: 488
  dev.igc.1.tx_abs_int_delay: 66
@@ -90,13 +90,13 @@
         CTRL_EXIT        10000040

  Interrupt Registers
-       ICR      00000001
+       ICR      00000000

  RX Registers
         RCTL     0440801e
         RDLEN    00004000
-       RDH      0000020a
-       RDT      00000209
+       RDH      000000e5
+       RDT      000000e4
         RXDCTL   02040808
         RDBAL    02f44000
         RDBAH    00000000
@@ -106,8 +106,8 @@
         TDBAL    02f06000
         TDBAH    00000000
         TDLEN    00004000
-       TDH      000000d2
-       TDT      000000d2
+       TDH      00000011
+       TDT      00000011
         TXDCTL   0201011f
         TDFH     00000000
         TDFT     00000000
@@ -119,38 +119,38 @@
  dev.igc.1.debug: -1
  dev.igc.1.nvm: -1
  dev.igc.1.iflib.rxq3.rxq_fl0.buf_size: 2048
-dev.igc.1.iflib.rxq3.rxq_fl0.credits: 1023
-dev.igc.1.iflib.rxq3.rxq_fl0.cidx: 908
-dev.igc.1.iflib.rxq3.rxq_fl0.pidx: 907
+dev.igc.1.iflib.rxq3.rxq_fl0.credits: 128
+dev.igc.1.iflib.rxq3.rxq_fl0.cidx: 0
+dev.igc.1.iflib.rxq3.rxq_fl0.pidx: 128
  dev.igc.1.iflib.rxq3.cpu: 3
  dev.igc.1.iflib.rxq2.rxq_fl0.buf_size: 2048
  dev.igc.1.iflib.rxq2.rxq_fl0.credits: 1023
-dev.igc.1.iflib.rxq2.rxq_fl0.cidx: 827
-dev.igc.1.iflib.rxq2.rxq_fl0.pidx: 826
+dev.igc.1.iflib.rxq2.rxq_fl0.cidx: 723
+dev.igc.1.iflib.rxq2.rxq_fl0.pidx: 722
  dev.igc.1.iflib.rxq2.cpu: 2
  dev.igc.1.iflib.rxq1.rxq_fl0.buf_size: 2048
  dev.igc.1.iflib.rxq1.rxq_fl0.credits: 1023
-dev.igc.1.iflib.rxq1.rxq_fl0.cidx: 414
-dev.igc.1.iflib.rxq1.rxq_fl0.pidx: 413
+dev.igc.1.iflib.rxq1.rxq_fl0.cidx: 84
+dev.igc.1.iflib.rxq1.rxq_fl0.pidx: 83
  dev.igc.1.iflib.rxq1.cpu: 1
  dev.igc.1.iflib.rxq0.rxq_fl0.buf_size: 2048
  dev.igc.1.iflib.rxq0.rxq_fl0.credits: 1023
-dev.igc.1.iflib.rxq0.rxq_fl0.cidx: 522
-dev.igc.1.iflib.rxq0.rxq_fl0.pidx: 521
+dev.igc.1.iflib.rxq0.rxq_fl0.cidx: 229
+dev.igc.1.iflib.rxq0.rxq_fl0.pidx: 228
  dev.igc.1.iflib.rxq0.cpu: 0
  dev.igc.1.iflib.txq3.r_abdications: 0
-dev.igc.1.iflib.txq3.r_restarts: 185763
-dev.igc.1.iflib.txq3.r_stalls: 185763
-dev.igc.1.iflib.txq3.r_starts: 22126656
+dev.igc.1.iflib.txq3.r_restarts: 0
+dev.igc.1.iflib.txq3.r_stalls: 0
+dev.igc.1.iflib.txq3.r_starts: 104312
  dev.igc.1.iflib.txq3.r_drops: 0
-dev.igc.1.iflib.txq3.r_enqueues: 23334820
-dev.igc.1.iflib.txq3.ring_state: pidx_head: 0430 pidx_tail: 0430 cidx: 
0430 state: IDLE
-dev.igc.1.iflib.txq3.txq_cleaned: 106314707
-dev.igc.1.iflib.txq3.txq_processed: 106314747
+dev.igc.1.iflib.txq3.r_enqueues: 104513
+dev.igc.1.iflib.txq3.ring_state: pidx_head: 1960 pidx_tail: 1960 cidx: 
1960 state: IDLE
+dev.igc.1.iflib.txq3.txq_cleaned: 208578
+dev.igc.1.iflib.txq3.txq_processed: 208618
  dev.igc.1.iflib.txq3.txq_in_use: 44
-dev.igc.1.iflib.txq3.txq_cidx_processed: 1019
-dev.igc.1.iflib.txq3.txq_cidx: 979
-dev.igc.1.iflib.txq3.txq_pidx: 1023
+dev.igc.1.iflib.txq3.txq_cidx_processed: 746
+dev.igc.1.iflib.txq3.txq_cidx: 706
+dev.igc.1.iflib.txq3.txq_pidx: 750
  dev.igc.1.iflib.txq3.no_tx_dma_setup: 0
  dev.igc.1.iflib.txq3.txd_encap_efbig: 0
  dev.igc.1.iflib.txq3.tx_map_failed: 0
@@ -160,18 +160,18 @@
  dev.igc.1.iflib.txq3.mbuf_defrag: 0
  dev.igc.1.iflib.txq3.cpu: 3
  dev.igc.1.iflib.txq2.r_abdications: 0
-dev.igc.1.iflib.txq2.r_restarts: 360876
-dev.igc.1.iflib.txq2.r_stalls: 360877
-dev.igc.1.iflib.txq2.r_starts: 24509109
+dev.igc.1.iflib.txq2.r_restarts: 0
+dev.igc.1.iflib.txq2.r_stalls: 0
+dev.igc.1.iflib.txq2.r_starts: 99386
  dev.igc.1.iflib.txq2.r_drops: 0
-dev.igc.1.iflib.txq2.r_enqueues: 26698302
-dev.igc.1.iflib.txq2.ring_state: pidx_head: 0070 pidx_tail: 0070 cidx: 
1883 state: STALLED
-dev.igc.1.iflib.txq2.txq_cleaned: 96507779
-dev.igc.1.iflib.txq2.txq_processed: 96507819
-dev.igc.1.iflib.txq2.txq_in_use: 987
-dev.igc.1.iflib.txq2.txq_cidx_processed: 939
-dev.igc.1.iflib.txq2.txq_cidx: 899
-dev.igc.1.iflib.txq2.txq_pidx: 862
+dev.igc.1.iflib.txq2.r_enqueues: 99386
+dev.igc.1.iflib.txq2.ring_state: pidx_head: 0394 pidx_tail: 0394 cidx: 
0394 state: IDLE
+dev.igc.1.iflib.txq2.txq_cleaned: 198730
+dev.igc.1.iflib.txq2.txq_processed: 198770
+dev.igc.1.iflib.txq2.txq_in_use: 42
+dev.igc.1.iflib.txq2.txq_cidx_processed: 114
+dev.igc.1.iflib.txq2.txq_cidx: 74
+dev.igc.1.iflib.txq2.txq_pidx: 116
  dev.igc.1.iflib.txq2.no_tx_dma_setup: 0
  dev.igc.1.iflib.txq2.txd_encap_efbig: 0
  dev.igc.1.iflib.txq2.tx_map_failed: 0
@@ -181,18 +181,18 @@
  dev.igc.1.iflib.txq2.mbuf_defrag: 0
  dev.igc.1.iflib.txq2.cpu: 2
  dev.igc.1.iflib.txq1.r_abdications: 0
-dev.igc.1.iflib.txq1.r_restarts: 1133594
-dev.igc.1.iflib.txq1.r_stalls: 1133594
-dev.igc.1.iflib.txq1.r_starts: 19099238
+dev.igc.1.iflib.txq1.r_restarts: 0
+dev.igc.1.iflib.txq1.r_stalls: 0
+dev.igc.1.iflib.txq1.r_starts: 751030
  dev.igc.1.iflib.txq1.r_drops: 0
-dev.igc.1.iflib.txq1.r_enqueues: 26635148
-dev.igc.1.iflib.txq1.ring_state: pidx_head: 0088 pidx_tail: 0088 cidx: 
0088 state: IDLE
-dev.igc.1.iflib.txq1.txq_cleaned: 134333877
-dev.igc.1.iflib.txq1.txq_processed: 134333917
-dev.igc.1.iflib.txq1.txq_in_use: 771
-dev.igc.1.iflib.txq1.txq_cidx_processed: 477
-dev.igc.1.iflib.txq1.txq_cidx: 437
-dev.igc.1.iflib.txq1.txq_pidx: 184
+dev.igc.1.iflib.txq1.r_enqueues: 751120
+dev.igc.1.iflib.txq1.ring_state: pidx_head: 0145 pidx_tail: 0145 cidx: 
0145 state: IDLE
+dev.igc.1.iflib.txq1.txq_cleaned: 6523185
+dev.igc.1.iflib.txq1.txq_processed: 6523225
+dev.igc.1.iflib.txq1.txq_in_use: 42
+dev.igc.1.iflib.txq1.txq_cidx_processed: 345
+dev.igc.1.iflib.txq1.txq_cidx: 305
+dev.igc.1.iflib.txq1.txq_pidx: 347
  dev.igc.1.iflib.txq1.no_tx_dma_setup: 0
  dev.igc.1.iflib.txq1.txd_encap_efbig: 0
  dev.igc.1.iflib.txq1.tx_map_failed: 0
@@ -202,18 +202,18 @@
  dev.igc.1.iflib.txq1.mbuf_defrag: 0
  dev.igc.1.iflib.txq1.cpu: 1
  dev.igc.1.iflib.txq0.r_abdications: 0
-dev.igc.1.iflib.txq0.r_restarts: 825923
-dev.igc.1.iflib.txq0.r_stalls: 825923
-dev.igc.1.iflib.txq0.r_starts: 31553019
+dev.igc.1.iflib.txq0.r_restarts: 0
+dev.igc.1.iflib.txq0.r_stalls: 0
+dev.igc.1.iflib.txq0.r_starts: 8
  dev.igc.1.iflib.txq0.r_drops: 0
-dev.igc.1.iflib.txq0.r_enqueues: 37045348
-dev.igc.1.iflib.txq0.ring_state: pidx_head: 1471 pidx_tail: 1471 cidx: 
1471 state: IDLE
-dev.igc.1.iflib.txq0.txq_cleaned: 153006246
-dev.igc.1.iflib.txq0.txq_processed: 153006286
-dev.igc.1.iflib.txq0.txq_in_use: 44
-dev.igc.1.iflib.txq0.txq_cidx_processed: 206
-dev.igc.1.iflib.txq0.txq_cidx: 166
-dev.igc.1.iflib.txq0.txq_pidx: 210
+dev.igc.1.iflib.txq0.r_enqueues: 8
+dev.igc.1.iflib.txq0.ring_state: pidx_head: 0816 pidx_tail: 0816 cidx: 
0816 state: IDLE
+dev.igc.1.iflib.txq0.txq_cleaned: 0
+dev.igc.1.iflib.txq0.txq_processed: 15
+dev.igc.1.iflib.txq0.txq_in_use: 17
+dev.igc.1.iflib.txq0.txq_cidx_processed: 15
+dev.igc.1.iflib.txq0.txq_cidx: 0
+dev.igc.1.iflib.txq0.txq_pidx: 17
  dev.igc.1.iflib.txq0.no_tx_dma_setup: 0
  dev.igc.1.iflib.txq0.txd_encap_efbig: 0
  dev.igc.1.iflib.txq0.tx_map_failed: 0





Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?760d007e-0a19-7f29-ade6-69a81744902d>