Date: Sat, 4 Jan 2003 09:56:40 -0800 From: steve@Watt.COM (Steve Watt) To: stable@freebsd.org Subject: crash in softdeps code Message-ID: <200301041756.h04HufhW003847@wattres.Watt.COM>
next in thread | raw e-mail | index | archive | help
I had an ugly crash last night, during the heavy disk load time for my
box (all the cron jobs conflate right about 3am).
Based on my quick glance, it seems that something ate the fs structure,
or maybe the pointer is bad. The disk in question is
ad0: 73308MB <IBM-DTLA-307075> [148945/16/63] at ata0-master UDMA33
The system was last brought up to -STABLE about 1300 PST 21 Dec 2002, and
had been up ever since. Uname says:
FreeBSD wattres.Watt.COM 4.7-STABLE FreeBSD 4.7-STABLE #13: Sat Dec 21 19:44:11 PST 2002 root@wattres.Watt.COM:/usr/obj/usr/src/sys/WATTRES i386
Here's what I found from a quick grope with gdb. Core is available, just
ask.
Script started on Sat Jan 4 09:32:33 2003
steve has logged on ttyv1 from local.
wattres.Watt.COM:/usr/obj/usr/src/sys/WATTRES
(root@wattres) 1# gdb -k kernel.debug /var/crash/vmcore.2
GNU gdb 4.18 (FreeBSD)
Copyright 1998 Free Software Foundation, Inc.
GDB is free software, covered by the GNU General Public License, and you are
welcome to change it and/or distribute copies of it under certain conditions.
Type "show copying" to see the conditions.
There is absolutely no warranty for GDB. Type "show warranty" for details.
This GDB was configured as "i386-unknown-freebsd"...Deprecated bfd_read called at /usr/src/gnu/usr.bin/binutils/gdb/../../../../contrib/gdb/gdb/dbxread.c line 2627 in elfstab_build_psymtabs
Deprecated bfd_read called at /usr/src/gnu/usr.bin/binutils/gdb/../../../../contrib/gdb/gdb/dbxread.c line 933 in fill_symbuf
IdlePTD at phsyical address 0x00498000
initial pcb at physical address 0x003e52c0
panicstr: ffs_blkfree: bad size
panic messages:
---
panic: ffs_blkfree: bad size
syncing disks... 6 3 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2
giving up on 2 buffers
Uptime: 13d6h33m56s
dumping to dev #da/1, offset 282200
dump 127 126 125 124 123 122 121 120 119 118 117 116 115 114 113 112 111 110 109 108 107 106 105 104 103 102 101 100 99 98 97 96 95 94 93 92 91 90 89 88 87 86 85 84 83 82 81 80 79 78 77 76 75 74 73 72 71 70 69 68 67 66 65 64 63 62 61 60 59 58 57 56 55 54 53 52 51 50 49 48 47 46 45 44 43 42 41 40 39 38 37 36 35 34 33 32 31 30 29 28 27 26 25 24 23 22 21 20 19 18 17 16 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1 0
---
#0 dumpsys () at /usr/src/sys/kern/kern_shutdown.c:487
487 if (dumping++) {
(kgdb) info sta
#0 dumpsys () at /usr/src/sys/kern/kern_shutdown.c:487
#1 0xc01f1bf0 in boot (howto=256) at /usr/src/sys/kern/kern_shutdown.c:316
#2 0xc01f2024 in poweroff_wait (junk=0xc0372c35, howto=-1070126080)
at /usr/src/sys/kern/kern_shutdown.c:595
#3 0xc02b5e11 in ffs_blkfree (ip=0xc8a35e0c, bno=-1459815749, size=8192)
at /usr/src/sys/ufs/ffs/ffs_alloc.c:1415
#4 0xc02ba9b6 in indir_trunc (ip=0xc8a35e0c, dbn=9832624, level=0, lbn=10252,
countp=0xc8a35dfc) at /usr/src/sys/ufs/ffs/ffs_softdep.c:2233
#5 0xc02ba99c in indir_trunc (ip=0xc8a35e0c, dbn=9963600, level=1, lbn=2060,
countp=0xc8a35dfc) at /usr/src/sys/ufs/ffs/ffs_softdep.c:2229
#6 0xc02ba756 in handle_workitem_freeblocks (freeblks=0xc1753780)
at /usr/src/sys/ufs/ffs/ffs_softdep.c:2133
#7 0xc02b8bf6 in process_worklist_item (matchmnt=0x0, flags=0)
at /usr/src/sys/ufs/ffs/ffs_softdep.c:723
#8 0xc02b8a82 in softdep_process_worklist (matchmnt=0x0)
at /usr/src/sys/ufs/ffs/ffs_softdep.c:622
#9 0xc022030b in sched_sync () at /usr/src/sys/kern/vfs_subr.c:1177
(kgdb) up
#1 0xc01f1bf0 in boot (howto=256) at /usr/src/sys/kern/kern_shutdown.c:316
316 dumpsys();
(kgdb) up
#2 0xc01f2024 in poweroff_wait (junk=0xc0372c35, howto=-1070126080)
at /usr/src/sys/kern/kern_shutdown.c:595
595 boot(bootopt);
(kgdb) up
#3 0xc02b5e11 in ffs_blkfree (ip=0xc8a35e0c, bno=-1459815749, size=8192)
at /usr/src/sys/ufs/ffs/ffs_alloc.c:1415
1415 panic("ffs_blkfree: bad size");
(kgdb) list
1410 if ((u_int)size > fs->fs_bsize || fragoff(fs, size) != 0 ||
1411 fragnum(fs, bno) + numfrags(fs, size) > fs->fs_frag) {
1412 printf("dev=%s, bno = %ld, bsize = %ld, size = %ld, fs = %s\n",
1413 devtoname(ip->i_dev), (long)bno, (long)fs->fs_bsize, size,
1414 fs->fs_fsmnt);
1415 panic("ffs_blkfree: bad size");
1416 }
1417 cg = dtog(fs, bno);
1418 if ((u_int)bno >= fs->fs_size) {
1419 printf("bad block %ld, ino %lu\n",
(kgdb) print ip->i_dev
$1 = 0xc1015d80
(kgdb) print bno
$2 = -1459815749
(kgdb) print *fs
$5 = {fs_firstfield = 1601398374, fs_unused_1 = 1718316130,
fs_sblkno = 979723634, fs_cblkno = 1684103712, fs_iblkno = 2053731104,
fs_dblkno = 1633812581, fs_cgoffset = 1818370148, fs_cgmask = 543908719,
fs_time = 744778789, fs_size = 1869506848, fs_dsize = 1970021664,
fs_ncg = 1633812490, fs_bsize = 1818370148, fs_fsize = 7037807, fs_frag = 0,
fs_minfree = 0, fs_rotdelay = 0, fs_rps = 0, fs_bmask = 1677721600,
fs_fmask = 1025537637, fs_bshift = 745743648, fs_fshift = 1869373984,
fs_maxcontig = 1025534819, fs_maxbpg = 1684808992,
fs_fragshift = 1936072748, fs_fsbtodb = 622869792, fs_sbsize = 1711278707,
fs_csmask = 1650422630, fs_csshift = 1919314796, fs_nindir = 540697957,
fs_inopb = 1701147238, fs_nspf = 543649385, fs_optim = 1701147238,
fs_npsect = 1869373984, fs_interleave = 1711303523,
fs_trackskew = 1650422630, fs_id = {1919314796, 540697957},
fs_csaddr = 1701147238, fs_cssize = 543649385, fs_cgsize = 1701147238,
fs_ntrak = 1634887200, fs_nsect = 1711276135, fs_spc = 1985966950,
fs_ncyl = 1701147238, fs_cpg = 1634869306, fs_ipg = 979724142,
fs_fpg = 1986356256, fs_cstotal = {cs_ndir = 673201440,
cs_nbfree = 623666213, cs_nifree = 539765092, cs_nffree = 544173673},
fs_fmod = 61 '=', fs_clean = 32 ' ', fs_ronly = 37 '%', fs_flags = 100 'd',
fs_fsmnt = ", fs = %s\000dev = %s, ino = %lu, fs = %s\n\000ffs_vfree: freeing free inode", '\000' <repeats 18 times>, "start = %d, len = %d, fs = %s\n\000ffs_alloccg: map corrupted\000bno = %lu, fs = %s\n\000ffs_alloccg: block not in map\000-", '\000' <repeats 19 times>, "pi"..., fs_cgrotor = -872415125, fs_ocsp = {
0xdcc03b97, 0x62c03b97, 0x6661736d, 0x70616d65, 0x2c000000, 0x3cc03b98,
---Type <return> to continue, or q <return> to quit---
0x61c03b98, 0x636f6c6c, 0x65726964, 0x8c007463, 0x9cc03b98, 0x69c03b98,
0x7269646e, 0x706564, 0xec000000, 0xfcc03b98, 0x61c03b98, 0x636f6c6c,
0x69646e69, 0x4c000072, 0x5cc03b99, 0x66c03b99, 0x66656572, 0x676172,
0xac000000, 0xbcc03b99, 0x66c03b99, 0x62656572, 0x736b6c},
fs_contigdirs = 0xc000000 <Address 0xc000000 out of bounds>,
fs_csp = 0x1cc03b9a, fs_maxcluster = 0x66c03b9a, fs_cpc = 1717921138,
fs_opostbl = {{27753, 101, 0, 27648, 15258, 31936, 15258, 25792}, {29289,
25697, 100, -13312, 15258, -9024, 15258, 28096}, {25707, 29289, 0,
11264, 15259, 15552, 15259, 25792}, {29289, 25970, 109, -29696, 15259,
-25408, 15259, 192}, {0, 0, 0, 0, 0, 0, 0, 29440}, {26223, 25716, 28773,
27743, 25455, 14955, 27680, 25455}, {26987, 26478, 24864, 24935, 28265,
29811, 27936, 29561}, {27749, 102, 28531, 29798, 25956, 24432, 28524,
27491}, {8250, 28524, 27491, 26656, 27749, 8292, 31074, 9504}, {100,
28531, 29798, 25956, 24432, 28277, 28524, 27491}, {8250, 28524, 27491,
28192, 29807, 26656, 27749, 100}, {0, 0, 0, 0, 0, 0, 0, 0}, {0, 0, 0, 0,
0, 0, 0, 26880}, {29806, 29285, 28524, 27491, 25701, 29535, 25964,
28773}, {8250, 28524, 27491, 28192, 29807, 26656, 27749, 100}, {0, 0, 0,
0, 0, 0, 0, 0}}, fs_snapinum = {0, 0, 0, 1761607680, 1919251566,
1801678700, 1935631461, 1885693292, 1853169722, 2003791467, 1886330990,
1952543333, 7237481, 0, 0, 0, 0, 0, 0, 1761607680},
fs_avgfilesize = 1919251566, fs_avgfpdir = 1801678700, fs_sparecon = {
1935631461, 1885693292, 1869357114, 1852402531, 1734418535, 1936615777,
1702043764, 26220, 0, 0, 0, 0, 0, 1761607680, 1919251566, 1801678700,
1935631461, 1885693292, 1869357114, 1746955107, 543452261, 622885218,
---Type <return> to continue, or q <return> to quit---
1702035556, 1918853485, 1634036837, 540697971},
fs_pendingblocks = 544501614, fs_pendinginodes = 1684825448,
fs_contigsumsize = 0, fs_maxsymlinklen = 1996488704,
fs_inodefmt = 1818980975, fs_maxfilesize = 7310308019004535657,
fs_qbmask = 7738151094940497010, fs_qfmask = 7810763617345695264,
fs_state = 1996488804, fs_postblformat = 1818980975, fs_nrpos = 1601467241,
fs_postbloff = 1702063721, fs_rotbloff = 540701810, fs_magic = 1701997665,
fs_space = "a"}
(kgdb) up
#4 0xc02ba9b6 in indir_trunc (ip=0xc8a35e0c, dbn=9832624, level=0, lbn=10252,
countp=0xc8a35dfc) at /usr/src/sys/ufs/ffs/ffs_softdep.c:2233
2233 ffs_blkfree(ip, nb, fs->fs_bsize);
(kgdb) list
2228 if (level != 0) {
2229 if ((error = indir_trunc(ip, fsbtodb(fs, nb),
2230 level - 1, lbn + (i * lbnadd), countp)) != 0)
2231 allerror = error;
2232 }
2233 ffs_blkfree(ip, nb, fs->fs_bsize);
2234 *countp += nblocks;
2235 }
2236 bp->b_flags |= B_INVAL | B_NOCACHE;
2237 brelse(bp);
(kgdb) q
wattres.Watt.COM:/usr/obj/usr/src/sys/WATTRES
(root@wattres) 3# ps -N /var/crash/kernel.2 -M /var/crash/vmcore.2 -axlww
UID PID PPID CPU PRI NI VSZ RSS WCHAN STAT TT TIME COMMAND
0 268 1 0 2 0 2932 0 sbwait I #C6- 0:00.03 (rmserver)
0 317 1 1 10 0 1260 0 wait Is #C6 0:00.04 (login)
0 328 317 0 3 0 1336 0 ttyin I+ #C6 0:00.05 (csh)
200 372 366 0 18 0 1328 0 pause Is #C6 0:00.08 (tcsh)
0 1095 372 1 3 0 1388 0 ttyin I+ #C6 0:00.67 (tcsh)
0 324 1 1 3 0 960 0 ttyin Is+ #C7 0:00.01 (getty)
0 323 1 1 3 0 960 0 ttyin Is+ #C7 0:00.01 (getty)
0 322 1 0 3 0 960 0 ttyin Is+ #C7 0:00.01 (getty)
0 321 1 0 3 0 960 0 ttyin Is+ #C7 0:00.01 (getty)
0 320 1 0 3 0 960 0 ttyin Is+ #C7 0:00.01 (getty)
0 319 1 0 3 0 960 0 ttyin Is+ #C7 0:00.01 (getty)
0 318 1 0 10 0 1260 0 wait Is #C7 0:00.03 (login)
200 334 318 0 3 0 1336 0 ttyin I+ #C7 0:00.09 (tcsh)
200 357 1 0 2 0 4812 0 select I #C7 4:42.66 (Xvnc)
0 363 1 0 2 0 3232 0 select I #C7 0:00.08 (xterm)
0 364 1 0 2 0 3280 0 select I #C7 0:05.64 (xterm)
0 365 1 0 2 0 3280 0 select I #C7 0:20.97 (xterm)
0 366 1 0 2 0 3360 0 select I #C7 0:08.34 (xterm)
0 367 1 0 2 0 3288 0 select I #C7 0:06.43 (xterm)
200 368 1 0 2 0 2504 0 select I #C7 0:00.90 (twm)
200 19799 19798 0 3 0 1316 0 ttyin Is+ #C1 0:00.08 (tcsh)
200 370 363 88 3 0 1332 0 ttyin Is+ #C1 0:00.06 (tcsh)
200 371 364 0 18 0 1316 0 pause Is #C1 0:00.05 (tcsh)
200 5096 371 0 2 0 2140 0 select I+ #C1 0:04.66 (ssh)
200 377 365 0 3 0 1352 0 ttyin Is+ #C1 0:00.59 (tcsh)
200 369 367 0 3 0 1380 0 ttyin Is+ #C1 0:00.76 (tcsh)
0 0 0 0 -18 0 0 0 sched DLs ?? 0:00.00 (swapper)
0 1 0 0 10 0 548 0 wait ILs ?? 0:00.78 (init)
0 2 0 0 -18 0 0 0 - RL ?? 6:28.28 (pagedaemon)
0 3 0 0 18 0 0 0 psleep DL ?? 0:00.00 (vmdaemon)
0 4 0 0 -18 0 0 0 - RL ?? 2:06.29 (bufdaemon)
0 5 0 264 -6 0 0 0 - RL ?? 18:46.12 (syncer)
0 6 0 0 -2 0 0 0 - RL ?? 0:04.62 (vnlru)
0 104 1 0 2 0 956 0 - Rs ?? 0:24.84 (syslogd)
0 108 1 0 2 0 3616 0 select Is ?? 3:17.89 (named)
0 110 1 0 2 0 1320 0 - Rs ?? 1:25.01 (ntpd)
0 118 1 0 2 0 1176 0 select Is ?? 0:00.57 (inetd)
0 120 1 0 10 0 1000 0 - Rs ?? 0:23.70 (cron)
0 123 1 0 2 0 972 0 select Is ?? 0:00.02 (lpd)
0 125 1 9 2 0 2360 0 select Is ?? 0:11.33 (sshd)
0 197 1 5 2 0 1008 0 accept I ?? 0:03.06 (netacl)
66 199 1 0 10 0 1012 0 - Rs ?? 0:14.80 (smapd)
8 201 1 0 2 0 1376 0 accept I ?? 0:01.42 (nntpd)
0 218 1 0 2 0 8932 0 - Rs ?? 0:35.06 (httpd)
66 222 1 115 2 0 2176 0 select Is ?? 0:00.02 (hfaxd)
80 229 218 0 2 0 9176 0 accept I ?? 0:01.43 (httpd)
80 230 218 0 2 0 9200 0 accept I ?? 0:01.74 (httpd)
80 231 218 0 2 0 9020 0 accept I ?? 0:01.38 (httpd)
80 232 218 0 2 0 9188 0 accept I ?? 0:01.65 (httpd)
80 233 218 0 2 0 9188 0 accept I ?? 0:01.49 (httpd)
66 235 1 0 2 0 2212 0 select Is ?? 0:00.01 (faxq)
0 241 1 0 2 0 1084 0 select Is ?? 0:00.29 (dhcpd)
0 265 1 0 2 0 2316 0 - Rs ?? 2:11.69 (racoon)
0 272 268 0 2 0 1062308 0 select Is ?? 0:00.59 (rmserver)
0 273 268 0 10 0 2932 0 nanslp Is ?? 0:14.39 (rmserver)
0 274 272 0 10 0 1051508 0 - R ?? 58:45.65 (rmserver)
0 284 1 1 2 0 2736 0 select Is ?? 0:05.90 (smbd)
0 286 1 0 2 0 1920 0 - Rs ?? 0:37.31 (nmbd)
0 288 286 0 -6 0 1892 0 piperd I ?? 0:00.10 (nmbd)
0 291 272 0 2 0 1051516 0 - R ?? 3:02.16 (rmserver)
0 292 272 0 2 0 1062308 0 select I ?? 0:00.01 (rmserver)
0 293 272 0 2 0 1062308 0 select I ?? 0:00.01 (rmserver)
0 294 272 0 2 0 1062308 0 select I ?? 0:00.02 (rmserver)
0 295 272 0 2 0 1062308 0 select I ?? 0:00.00 (rmserver)
0 296 272 0 2 0 1062308 0 select I ?? 0:00.00 (rmserver)
0 297 272 0 2 0 1062308 0 select I ?? 0:00.01 (rmserver)
0 298 272 0 2 0 1062308 0 select I ?? 0:00.01 (rmserver)
0 299 272 0 2 0 1062308 0 select I ?? 0:00.02 (rmserver)
0 300 272 0 2 0 1062316 0 select I ?? 0:00.14 (rmserver)
0 301 272 0 2 0 1062308 0 select I ?? 0:00.01 (rmserver)
0 302 272 0 2 0 1062308 0 select I ?? 0:00.00 (rmserver)
0 303 272 0 2 0 1062308 0 select I ?? 0:00.01 (rmserver)
0 304 272 0 2 0 1062308 0 select I ?? 0:00.02 (rmserver)
0 305 272 0 2 0 1062308 0 select I ?? 0:00.00 (rmserver)
0 306 272 0 2 0 1062308 0 select I ?? 0:00.07 (rmserver)
0 307 272 0 2 0 1062308 0 select I ?? 0:00.01 (rmserver)
0 308 272 0 2 0 1062308 0 select I ?? 0:00.00 (rmserver)
0 309 272 0 2 0 1062436 0 - R ?? 0:48.43 (rmserver)
0 313 1 0 18 0 2832 0 pause Is ?? 0:01.03 (sendmail)
0 326 1 0 3 0 956 0 siodcd I ?? 0:00.01 (getty)
66 327 1 0 2 0 2332 0 select I ?? 0:00.03 (faxgetty)
0 5015 1 0 2 0 1972 0 select Is ?? 0:00.01 (mpd)
80 8857 218 0 2 0 9024 0 accept I ?? 0:01.32 (httpd)
80 9818 218 0 2 0 9168 0 accept I ?? 0:01.40 (httpd)
80 11054 218 0 2 0 9180 0 accept I ?? 0:01.35 (httpd)
0 19798 125 0 2 0 2472 0 select I ?? 0:01.34 (sshd)
0 25507 118 0 2 0 2860 0 - Rs ?? 1:41.31 (ppp)
80 34640 218 0 2 0 9192 0 accept I ?? 0:01.74 (httpd)
80 34641 218 0 2 0 9192 0 accept I ?? 0:01.45 (httpd)
0 38922 1 0 3 0 956 0 siodcd I ?? 0:00.01 (getty)
204 66059 118 0 2 0 3492 0 select Is ?? 0:18.65 (imapd)
8 77775 201 0 2 0 1480 0 - R ?? 0:00.36 (nntpd)
8 78969 201 0 2 0 1480 0 - R ?? 0:00.68 (nntpd)
8 80465 201 0 2 0 1480 0 - R ?? 0:01.48 (nntpd)
0 81206 120 0 -6 0 1032 0 piperd I ?? 0:00.00 (cron)
200 81210 81206 0 10 0 632 0 wait Is ?? 0:00.00 (sh)
200 81213 81210 0 28 0 7736 0 - R ?? 92:39.61 (glimpseindex)
0 81231 81206 0 -6 0 2832 0 piperd Is ?? 0:00.06 (sendmail)
200 86099 118 0 2 0 3152 0 select Is ?? 0:06.80 (imapd)
8 98329 201 0 2 0 1480 0 - R ?? 0:00.81 (nntpd)
wattres.Watt.COM:/usr/obj/usr/src/sys/WATTRES
(root@wattres) 4# exit
Script done on Sat Jan 4 09:37:08 2003
--
Steve Watt KD6GGD PP-ASEL-IA ICBM: 121W 56' 57.8" / 37N 20' 14.9"
Internet: steve @ Watt.COM Whois: SW32
Free time? There's no such thing. It just comes in varying prices...
To Unsubscribe: send mail to majordomo@FreeBSD.org
with "unsubscribe freebsd-stable" in the body of the message
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?200301041756.h04HufhW003847>