Date: Tue, 13 Sep 2011 17:01:21 +0200 From: Bartosz Stec <bartosz.stec@it4pro.pl> To: "Vogel, Jack" <jack.vogel@intel.com> Cc: FreeBSD Stable <freebsd-stable@FreeBSD.org>, Jeremy Chadwick <freebsd@jdc.parodius.com> Subject: Re: Panic during kernel booting on HP Proliant DL180G6 and latest STABLE Message-ID: <4E6F7041.6040602@it4pro.pl> In-Reply-To: <1DB50624F8348F48840F2E2CF6040A9D018D22E697@orsmsx508.amr.corp.intel.com> References: <4DC6A277.4030801@it4pro.pl> <4DC6E23B.2040207@it4pro.pl> <4DC81E22.5030806@it4pro.pl> <4DCC3844.6070008@it4pro.pl> <4DCD1672.4020405@FreeBSD.org> <4DF1DC26.5010600@it4pro.pl> <20110610093728.GA39453@icarus.home.lan> <4DF1EB98.9080109@it4pro.pl> <1DB50624F8348F48840F2E2CF6040A9D018ACA75C7@orsmsx508.amr.corp.intel.com> <4DFB59A6.9030508@it4pro.pl> <1DB50624F8348F48840F2E2CF6040A9D018D22E697@orsmsx508.amr.corp.intel.com>
next in thread | previous in thread | raw e-mail | index | archive | help
Hi again list. Here's little update about the issue.
> Yes, there have been a few times when I have seen something like this, and issue when the driver is static and not when modulized. I have never been clear what to make of this. If anyone else here has any thoughts or enlightening ideas I'd appreciate it.
Fresh build, 8.2-STABLE #7: Tue Sep 13 14:25:45, problem is still here,
regardless of igb included in the kernel or loaded as module from
loader.conf, system panic with msix enabled.
> Also if it does not even happen when using the GENERIC config then it would be interesting to see the diff between that and what does cause the panic.
Here it is:
--- GENERIC 2011-04-10 19:55:25.304366901 +0200
+++ PROLIANT 2011-06-18 16:03:32.728117341 +0200
@@ -19,7 +19,7 @@
# $FreeBSD: src/sys/amd64/conf/GENERIC,v 1.531.2.17 2011/04/09 12:04:35 bz Exp $
cpu HAMMER
-ident GENERIC
+ident PROLIANT
# To statically compile in device wiring instead of /boot/device.hints
#hints "GENERIC.hints" # Default places to look for devices.
@@ -30,7 +30,7 @@
#
# env "GENERIC.env"
-makeoptions DEBUG=-g # Build kernel with gdb(1) debug symbols
+#makeoptions DEBUG=-g # Build kernel with gdb(1) debug symbols
options SCHED_ULE # ULE scheduler
options PREEMPTION # Enable kernel thread preemption
@@ -44,9 +44,9 @@
options UFS_GJOURNAL # Enable gjournal-based UFS journaling
options MD_ROOT # MD is a potential root device
options NFSCLIENT # Network Filesystem Client
-options NFSSERVER # Network Filesystem Server
-options NFSLOCKD # Network Lock Manager
-options NFS_ROOT # NFS usable as /, requires NFSCLIENT
+#options NFSSERVER # Network Filesystem Server
+#options NFSLOCKD # Network Lock Manager
+#options NFS_ROOT # NFS usable as /, requires NFSCLIENT
options MSDOSFS # MSDOS Filesystem
options CD9660 # ISO 9660 Filesystem
options PROCFS # Process filesystem (requires PSEUDOFS)
@@ -81,10 +81,18 @@
# Make an SMP-capable kernel by default
options SMP # Symmetric MultiProcessor Kernel
+options DEVICE_POLLING
# CPU frequency control
device cpufreq
+options ALTQ
+options ALTQ_CBQ # Class Bases Queuing (CBQ)
+options ALTQ_RED # Random Early Detection (RED)
+options ALTQ_RIO # RED In/Out
+options ALTQ_HFSC # Hierarchical Packet Scheduler (HFSC)
+options ALTQ_PRIQ # Priority Queuing (PRIQ)
+
# Bus support.
device acpi
device pci
@@ -102,59 +110,59 @@
options ATA_STATIC_ID # Static device numbering
# SCSI Controllers
-device ahc # AHA2940 and onboard AIC7xxx devices
-options AHC_REG_PRETTY_PRINT # Print register bitfields in debug
+#device ahc # AHA2940 and onboard AIC7xxx devices
+#options AHC_REG_PRETTY_PRINT # Print register bitfields in debug
# output. Adds ~128k to driver.
-device ahd # AHA39320/29320 and onboard AIC79xx devices
-options AHD_REG_PRETTY_PRINT # Print register bitfields in debug
+#device ahd # AHA39320/29320 and onboard AIC79xx devices
+#options AHD_REG_PRETTY_PRINT # Print register bitfields in debug
# output. Adds ~215k to driver.
-device amd # AMD 53C974 (Tekram DC-390(T))
-device hptiop # Highpoint RocketRaid 3xxx series
-device isp # Qlogic family
+#device amd # AMD 53C974 (Tekram DC-390(T))
+#device hptiop # Highpoint RocketRaid 3xxx series
+#device isp # Qlogic family
#device ispfw # Firmware for QLogic HBAs- normally a module
-device mpt # LSI-Logic MPT-Fusion
-device mps # LSI-Logic MPT-Fusion 2
+#device mpt # LSI-Logic MPT-Fusion
+#device mps # LSI-Logic MPT-Fusion 2
#device ncr # NCR/Symbios Logic
-device sym # NCR/Symbios Logic (newer chipsets + those of `ncr')
-device trm # Tekram DC395U/UW/F DC315U adapters
+#device sym # NCR/Symbios Logic (newer chipsets + those of `ncr')
+#device trm # Tekram DC395U/UW/F DC315U adapters
-device adv # Advansys SCSI adapters
-device adw # Advansys wide SCSI adapters
-device aic # Adaptec 15[012]x SCSI adapters, AIC-6[23]60.
-device bt # Buslogic/Mylex MultiMaster SCSI adapters
+#device adv # Advansys SCSI adapters
+#device adw # Advansys wide SCSI adapters
+#device aic # Adaptec 15[012]x SCSI adapters, AIC-6[23]60.
+#device bt # Buslogic/Mylex MultiMaster SCSI adapters
# SCSI peripherals
device scbus # SCSI bus (required for SCSI)
-device ch # SCSI media changers
+#device ch # SCSI media changers
device da # Direct Access (disks)
-device sa # Sequential Access (tape etc)
+#device sa # Squential Access (tape etc)
device cd # CD
device pass # Passthrough device (direct SCSI access)
-device ses # SCSI Environmental Services (and SAF-TE)
+#device ses # SCSI Environmental Services (and SAF-TE)
# RAID controllers interfaced to the SCSI subsystem
-device amr # AMI MegaRAID
-device arcmsr # Areca SATA II RAID
+#device amr # AMI MegaRAID
+#device arcmsr # Areca SATA II RAID
#XXX it is not 64-bit clean, -scottl
#device asr # DPT SmartRAID V, VI and Adaptec SCSI RAID
-device ciss # Compaq Smart RAID 5*
-device dpt # DPT Smartcache III, IV - See NOTES for options
-device hptmv # Highpoint RocketRAID 182x
-device hptrr # Highpoint RocketRAID 17xx, 22xx, 23xx, 25xx
-device iir # Intel Integrated RAID
-device ips # IBM (Adaptec) ServeRAID
-device mly # Mylex AcceleRAID/eXtremeRAID
-device twa # 3ware 9000 series PATA/SATA RAID
+#device ciss # Compaq Smart RAID 5*
+#device dpt # DPT Smartcache III, IV - See NOTES for options
+#device hptmv # Highpoint RocketRAID 182x
+#device hptrr # Highpoint RocketRAID 17xx, 22xx, 23xx, 25xx
+#device iir # Intel Integrated RAID
+#device ips # IBM (Adaptec) ServeRAID
+#device mly # Mylex AcceleRAID/eXtremeRAID
+#device twa # 3ware 9000 series PATA/SATA RAID
# RAID controllers
-device aac # Adaptec FSA RAID
-device aacp # SCSI passthrough for aac (requires CAM)
-device ida # Compaq Smart RAID
-device mfi # LSI MegaRAID SAS
-device mlx # Mylex DAC960 family
+#device aac # Adaptec FSA RAID
+#device aacp # SCSI passthrough for aac (requires CAM)
+#device ida # Compaq Smart RAID
+#device mfi # LSI MegaRAID SAS
+#device mlx # Mylex DAC960 family
#XXX pointer/int warnings
#device pst # Promise Supertrak SX6000
-device twe # 3ware ATA RAID
+#device twe # 3ware ATA RAID
# atkbdc0 controls both the keyboard and the PS/2 mouse
device atkbdc # AT keyboard controller
@@ -170,13 +178,13 @@
# syscons is the default console driver, resembling an SCO console
device sc
-device agp # support several AGP chipsets
+#device agp # support several AGP chipsets
# PCCARD (PCMCIA) support
# PCMCIA and cardbus bridge support
-device cbb # cardbus (yenta) bridge
-device pccard # PC Card (16-bit) bus
-device cardbus # CardBus (32-bit) bus
+#device cbb # cardbus (yenta) bridge
+#device pccard # PC Card (16-bit) bus
+#device cardbus # CardBus (32-bit) bus
# Serial (COM) ports
device uart # Generic UART driver
@@ -195,76 +203,76 @@
#device puc
# PCI Ethernet NICs.
-device de # DEC/Intel DC21x4x (``Tulip'')
-device em # Intel PRO/1000 Gigabit Ethernet Family
-device igb # Intel PRO/1000 PCIE Server Gigabit Family
-device ixgbe # Intel PRO/10GbE PCIE Ethernet Family
-device le # AMD Am7900 LANCE and Am79C9xx PCnet
-device ti # Alteon Networks Tigon I/II gigabit Ethernet
-device txp # 3Com 3cR990 (``Typhoon'')
-device vx # 3Com 3c590, 3c595 (``Vortex'')
+#device de # DEC/Intel DC21x4x (``Tulip'')
+#device em # Intel PRO/1000 Gigabit Ethernet Family
+#device igb # Intel PRO/1000 PCIE Server Gigabit Family
+#device ixgbe # Intel PRO/10GbE PCIE Ethernet Family
+#device le # AMD Am7900 LANCE and Am79C9xx PCnet
+#device ti # Alteon Networks Tigon I/II gigabit Ethernet
+#device txp # 3Com 3cR990 (``Typhoon'')
+#device vx # 3Com 3c590, 3c595 (``Vortex'')
# PCI Ethernet NICs that use the common MII bus controller code.
# NOTE: Be sure to keep the 'device miibus' line in order to use these NICs!
device miibus # MII bus support
-device ae # Attansic/Atheros L2 FastEthernet
-device age # Attansic/Atheros L1 Gigabit Ethernet
-device alc # Atheros AR8131/AR8132 Ethernet
-device ale # Atheros AR8121/AR8113/AR8114 Ethernet
-device bce # Broadcom BCM5706/BCM5708 Gigabit Ethernet
-device bfe # Broadcom BCM440x 10/100 Ethernet
-device bge # Broadcom BCM570xx Gigabit Ethernet
-device dc # DEC/Intel 21143 and various workalikes
-device et # Agere ET1310 10/100/Gigabit Ethernet
-device fxp # Intel EtherExpress PRO/100B (82557, 82558)
-device jme # JMicron JMC250 Gigabit/JMC260 Fast Ethernet
-device lge # Level 1 LXT1001 gigabit Ethernet
-device msk # Marvell/SysKonnect Yukon II Gigabit Ethernet
-device nfe # nVidia nForce MCP on-board Ethernet
-device nge # NatSemi DP83820 gigabit Ethernet
+#device ae # Attansic/Atheros L2 FastEthernet
+#device age # Attansic/Atheros L1 Gigabit Ethernet
+#device alc # Atheros AR8131/AR8132 Ethernet
+#device ale # Atheros AR8121/AR8113/AR8114 Ethernet
+#device bce # Broadcom BCM5706/BCM5708 Gigabit Ethernet
+#device bfe # Broadcom BCM440x 10/100 Ethernet
+#device bge # Broadcom BCM570xx Gigabit Ethernet
+#device dc # DEC/Intel 21143 and various workalikes
+#device et # Agere ET1310 10/100/Gigabit Ethernet
+#device fxp # Intel EtherExpress PRO/100B (82557, 82558)
+#device jme # JMicron JMC250 Gigabit/JMC260 Fast Ethernet
+#device lge # Level 1 LXT1001 gigabit Ethernet
+#device msk # Marvell/SysKonnect Yukon II Gigabit Ethernet
+#device nfe # nVidia nForce MCP on-board Ethernet
+#device nge # NatSemi DP83820 gigabit Ethernet
#device nve # nVidia nForce MCP on-board Ethernet Networking
-device pcn # AMD Am79C97x PCI 10/100 (precedence over 'le')
-device re # RealTek 8139C+/8169/8169S/8110S
-device rl # RealTek 8129/8139
-device sf # Adaptec AIC-6915 (``Starfire'')
-device sge # Silicon Integrated Systems SiS190/191
-device sis # Silicon Integrated Systems SiS 900/SiS 7016
-device sk # SysKonnect SK-984x& SK-982x gigabit Ethernet
-device ste # Sundance ST201 (D-Link DFE-550TX)
-device stge # Sundance/Tamarack TC9021 gigabit Ethernet
-device tl # Texas Instruments ThunderLAN
-device tx # SMC EtherPower II (83c170 ``EPIC'')
-device vge # VIA VT612x gigabit Ethernet
-device vr # VIA Rhine, Rhine II
-device wb # Winbond W89C840F
-device xl # 3Com 3c90x (``Boomerang'', ``Cyclone'')
+#device pcn # AMD Am79C97x PCI 10/100 (precedence over 'le')
+#device re # RealTek 8139C+/8169/8169S/8110S
+#device rl # RealTek 8129/8139
+#device sf # Adaptec AIC-6915 (``Starfire'')
+#device sge # Silicon Integrated Systems SiS190/191
+#device sis # Silicon Integrated Systems SiS 900/SiS 7016
+#device sk # SysKonnect SK-984x& SK-982x gigabit Ethernet
+#device ste # Sundance ST201 (D-Link DFE-550TX)
+#device stge # Sundance/Tamarack TC9021 gigabit Ethernet
+#device tl # Texas Instruments ThunderLAN
+#device tx # SMC EtherPower II (83c170 ``EPIC'')
+#device vge # VIA VT612x gigabit Ethernet
+#device vr # VIA Rhine, Rhine II
+#device wb # Winbond W89C840F
+#device xl # 3Com 3c90x (``Boomerang'', ``Cyclone'')
# ISA Ethernet NICs. pccard NICs included.
-device cs # Crystal Semiconductor CS89x0 NIC
+#device cs # Crystal Semiconductor CS89x0 NIC
# 'device ed' requires 'device miibus'
-device ed # NE[12]000, SMC Ultra, 3c503, DS8390 cards
-device ex # Intel EtherExpress Pro/10 and Pro/10+
-device ep # Etherlink III based cards
-device fe # Fujitsu MB8696x based cards
-device sn # SMC's 9000 series of Ethernet chips
-device xe # Xircom pccard Ethernet
+#device ed # NE[12]000, SMC Ultra, 3c503, DS8390 cards
+#device ex # Intel EtherExpress Pro/10 and Pro/10+
+#device ep # Etherlink III based cards
+#device fe # Fujitsu MB8696x based cards
+#device sn # SMC's 9000 series of Ethernet chips
+#device xe # Xircom pccard Ethernet
# Wireless NIC cards
-device wlan # 802.11 support
-options IEEE80211_DEBUG # enable debug msgs
-options IEEE80211_AMPDU_AGE # age frames in AMPDU reorder q's
-options IEEE80211_SUPPORT_MESH # enable 802.11s draft support
-device wlan_wep # 802.11 WEP support
-device wlan_ccmp # 802.11 CCMP support
-device wlan_tkip # 802.11 TKIP support
-device wlan_amrr # AMRR transmit rate control algorithm
-device an # Aironet 4500/4800 802.11 wireless NICs.
-device ath # Atheros pci/cardbus NIC's
-device ath_hal # pci/cardbus chip support
-options AH_SUPPORT_AR5416 # enable AR5416 tx/rx descriptors
-device ath_rate_sample # SampleRate tx rate control for ath
-device ral # Ralink Technology RT2500 wireless NICs.
-device wi # WaveLAN/Intersil/Symbol 802.11 wireless NICs.
+#device wlan # 802.11 support
+#options IEEE80211_DEBUG # enable debug msgs
+#options IEEE80211_AMPDU_AGE # age frames in AMPDU reorder q's
+#options IEEE80211_SUPPORT_MESH # enable 802.11s draft support
+#device wlan_wep # 802.11 WEP support
+#device wlan_ccmp # 802.11 CCMP support
+#device wlan_tkip # 802.11 TKIP support
+#device wlan_amrr # AMRR transmit rate control algorithm
+#device an # Aironet 4500/4800 802.11 wireless NICs.
+#device ath # Atheros pci/cardbus NIC's
+#device ath_hal # pci/cardbus chip support
+#options AH_SUPPORT_AR5416 # enable AR5416 tx/rx descriptors
+#device ath_rate_sample # SampleRate tx rate control for ath
+#device ral # Ralink Technology RT2500 wireless NICs.
+#device wi # WaveLAN/Intersil/Symbol 802.11 wireless NICs.
# Pseudo devices.
device loop # Network loopback
@@ -295,34 +303,36 @@
device ulpt # Printer
device umass # Disks/Mass storage - Requires scbus and da
device ums # Mouse
-device urio # Diamond Rio 500 MP3 player
+#device urio # Diamond Rio 500 MP3 player
# USB Serial devices
-device uark # Technologies ARK3116 based serial adapters
-device ubsa # Belkin F5U103 and compatible serial adapters
-device uftdi # For FTDI usb serial adapters
-device uipaq # Some WinCE based devices
-device uplcom # Prolific PL-2303 serial adapters
-device uslcom # SI Labs CP2101/CP2102 serial adapters
-device uvisor # Visor and Palm devices
-device uvscom # USB serial support for DDI pocket's PHS
+#device uark # Technologies ARK3116 based serial adapters
+#device ubsa # Belkin F5U103 and compatible serial adapters
+#device uftdi # For FTDI usb serial adapters
+#device uipaq # Some WinCE based devices
+#device uplcom # Prolific PL-2303 serial adapters
+#device uslcom # SI Labs CP2101/CP2102 serial adapters
+#device uvisor # Visor and Palm devices
+#device uvscom # USB serial support for DDI pocket's PHS
# USB Ethernet, requires miibus
-device aue # ADMtek USB Ethernet
-device axe # ASIX Electronics USB Ethernet
-device cdce # Generic USB over Ethernet
-device cue # CATC USB Ethernet
-device kue # Kawasaki LSI USB Ethernet
-device rue # RealTek RTL8150 USB Ethernet
-device udav # Davicom DM9601E USB
+#device aue # ADMtek USB Ethernet
+#device axe # ASIX Electronics USB Ethernet
+#device cdce # Generic USB over Ethernet
+#device cue # CATC USB Ethernet
+#device kue # Kawasaki LSI USB Ethernet
+#device rue # RealTek RTL8150 USB Ethernet
+#device udav # Davicom DM9601E USB
# USB Wireless
-device rum # Ralink Technology RT2501USB wireless NICs
-device uath # Atheros AR5523 wireless NICs
-device ural # Ralink Technology RT2500USB wireless NICs
-device zyd # ZyDAS zb1211/zb1211b wireless NICs
+#device rum # Ralink Technology RT2501USB wireless NICs
+#device uath # Atheros AR5523 wireless NICs
+#device ural # Ralink Technology RT2500USB wireless NICs
+#device zyd # ZyDAS zb1211/zb1211b wireless NICs
# FireWire support
-device firewire # FireWire bus code
+#device firewire # FireWire bus code
#device sbp # SCSI over FireWire (Requires scbus and da)
-device fwe # Ethernet over FireWire (non-standard!)
-device fwip # IP over FireWire (RFC 2734,3146)
-device dcons # Dumb console driver
-device dcons_crom # Configuration ROM for dcons
+#device fwe # Ethernet over FireWire (non-standard!)
+#device fwip # IP over FireWire (RFC 2734,3146)
+#device dcons # Dumb console driver
+#device dcons_crom # Configuration ROM for dcons
+
+device atapicam
>
> -----Original Message-----
> From: Bartosz Stec [mailto:bartosz.stec@it4pro.pl]
> Sent: Friday, June 17, 2011 6:42 AM
> To: Vogel, Jack
> Cc: Jeremy Chadwick; FreeBSD Stable
> Subject: Re: Panic during kernel booting on HP Proliant DL180G6 and latest STABLE
>
> W dniu 2011-06-10 20:23, Vogel, Jack pisze:
>> Er, so what if you get rid of ZFS, does your panic go away? It doesn't really matter what type adapter it is, the igb driver only requests standard size clusters, so memory is getting trashed somewhere I suspect.
>>
>> Jack
> Well, from my observations about this issue (which could be very wrong
> because my lack of knowledge about BSD kernel) I don't suspect igb
> driver directly, but rather an order which kernel is processing stuff
> related to MSIX and hardware (so I suppose that real cause of the
> problem could be very hard to catch and repeat)? Here's why:
>
> 1. There's no panic when using GENERIC kernel. There's also nothing
> unusual in my custom kernel (included in thread), neither in make.conf.
>
> 2. Before current build, panic was seen with igb driver included in
> kernel, but no panic when using a module. Even better - no panic when
> trying to load a module while igb driver is stil included in source. No
> random memory corruption here - same scenario seen every boot with all
> variants above. It's HP server with HP ECC memory by the way.
>
> 3. With current build kernel panics regardless if igb driver is a module
> or included in kernel (unless i disable MSIX). But I found override - I
> removed igb driver from kernel config, and a module from loader.conf.
> Than booted in single user mode, and manually loaded igb driver. No
> panic! Appareantly something gets wrong _only_ at boot time.
>
It seems that I was right about the relation of this issue with booting
process. I enabled MSIX and, removed igb driver in loader.conf, and
added if_igb to rc.conf uder ' kld_list="" '.
No panic if igb driver is loaded after kernel booting is complete:
igb0: <Intel(R) PRO/1000 Network Connection version - 2.2.3> port
0xe880-0xe89f mem
0xfbe60000-0xfbe7ffff,0xfbe40000-0xfbe5ffff,0xfbeb8000-0xfbebbfff
irq 28 at device 0.0 on pci7
igb0: Using MSIX interrupts with 5 vectors
igb0: [ITHREAD]
igb0: [ITHREAD]
igb0: [ITHREAD]
igb0: [ITHREAD]
igb0: [ITHREAD]
igb0: Ethernet address: d8:d3:85:e3:e4:f6
igb1: <Intel(R) PRO/1000 Network Connection version - 2.2.3> port
0xec00-0xec1f mem
0xfbee0000-0xfbefffff,0xfbec0000-0xfbedffff,0xfbebc000-0xfbebffff
irq 40 at device 0.1 on pci7
igb1: Using MSIX interrupts with 5 vectors
igb1: [ITHREAD]
igb1: [ITHREAD]
igb1: [ITHREAD]
igb1: [ITHREAD]
igb1: [ITHREAD]
igb1: Ethernet address: d8:d3:85:e3:e4:f7
igb0: link state changed to UP
igb1: link state changed to UP
kld_list feature looks like great problem override, because now I I have
msix enabled and can reboot my server safelly, but it still far from
actual fix.
Any new ideas?
--
Bartosz Stec
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?4E6F7041.6040602>