Skip site navigation (1)Skip section navigation (2)
Date:      Fri, 25 Jan 2013 13:50:48 +0900
From:      YongHyeon PYUN <pyunyh@gmail.com>
To:        John Baldwin <jhb@freebsd.org>
Cc:        freebsd-fs@freebsd.org, Christian Gusenbauer <c47g@gmx.at>, yongari@freebsd.org
Subject:   Re: 9.1-stable crashes while copying data from a NFS mounted directory
Message-ID:  <20130125045048.GB1429@michelle.cdnetworks.com>
In-Reply-To: <20130125043043.GA1429@michelle.cdnetworks.com>
References:  <201301241805.57623.c47g@gmx.at> <201301242150.52238.c47g@gmx.at> <20130124212212.GM2522@kib.kiev.ua> <201301241721.51102.jhb@freebsd.org> <20130125043043.GA1429@michelle.cdnetworks.com>

next in thread | previous in thread | raw e-mail | index | archive | help

--8P1HSweYDcXXzwPJ
Content-Type: text/plain; charset=us-ascii
Content-Disposition: inline

On Fri, Jan 25, 2013 at 01:30:43PM +0900, YongHyeon PYUN wrote:
> On Thu, Jan 24, 2013 at 05:21:50PM -0500, John Baldwin wrote:
> > On Thursday, January 24, 2013 4:22:12 pm Konstantin Belousov wrote:
> > > On Thu, Jan 24, 2013 at 09:50:52PM +0100, Christian Gusenbauer wrote:
> > > > On Thursday 24 January 2013 20:37:09 Konstantin Belousov wrote:
> > > > > On Thu, Jan 24, 2013 at 07:50:49PM +0100, Christian Gusenbauer wrote:
> > > > > > On Thursday 24 January 2013 19:07:23 Konstantin Belousov wrote:
> > > > > > > On Thu, Jan 24, 2013 at 08:03:59PM +0200, Konstantin Belousov wrote:
> > > > > > > > On Thu, Jan 24, 2013 at 06:05:57PM +0100, Christian Gusenbauer wrote:
> > > > > > > > > Hi!
> > > > > > > > > 
> > > > > > > > > I'm using 9.1 stable svn revision 245605 and I get the panic below
> > > > > > > > > if I execute the following commands (as single user):
> > > > > > > > > 
> > > > > > > > > # swapon -a
> > > > > > > > > # dumpon /dev/ada0s3b
> > > > > > > > > # mount -u /
> > > > > > > > > # ifconfig age0 inet 192.168.2.2 mtu 6144 up
> > > > > > > > > # mount -t nfs -o rsize=32768 data:/multimedia /mnt
> > > > > > > > > # cp /mnt/Movies/test/a.m2ts /tmp
> > > > > > > > > 
> > > > > > > > > then the system panics almost immediately. I'll attach the stack
> > > > > > > > > trace.
> > > > > > > > > 
> > > > > > > > > Note, that I'm using jumbo frames (6144 byte) on a 1Gbit network,
> > > > > > > > > maybe that's the cause for the panic, because the bcopy (see stack
> > > > > > > > > frame #15) fails.
> > > > > > > > > 
> > > > > > > > > Any clues?
> > > > > > > > 
> > > > > > > > I tried a similar operation with the nfs mount of rsize=32768 and mtu
> > > > > > > > 6144, but the machine runs HEAD and em instead of age. I was unable
> > > > > > > > to reproduce the panic on the copy of the 5GB file from nfs mount.
> > > > > > 
> > > > > > Hmmm, I did a quick test. If I do not change the MTU, so just configuring
> > > > > > age0 with
> > > > > > 
> > > > > > # ifconfig age0 inet 192.168.2.2 up
> > > > > > 
> > > > > > then I can copy all files from the mounted directory without any
> > > > > > problems, too. So it's probably age0 related?
> > > > > 
> > > > > From your backtrace and the buffer printout, I see somewhat strange thing.
> > > > > The buffer data address is 0xffffff8171418000, while kernel faulted
> > > > > at the attempt to write at 0xffffff8171413000, which is is lower then
> > > > > the buffer data pointer, at the attempt to bcopy to the buffer.
> > > > > 
> > > > > The other data suggests that there were no overflow of the data from the
> > > > > server response. So it might be that mbuf_len(mp) returned negative number
> > > > > ? I am not sure is it possible at all.
> > > > > 
> > > > > Try this debugging patch, please. You need to add INVARIANTS etc to the
> > > > > kernel config.
> > > > > 
> > > > > diff --git a/sys/fs/nfs/nfs_commonsubs.c b/sys/fs/nfs/nfs_commonsubs.c
> > > > > index efc0786..9a6bda5 100644
> > > > > --- a/sys/fs/nfs/nfs_commonsubs.c
> > > > > +++ b/sys/fs/nfs/nfs_commonsubs.c
> > > > > @@ -218,6 +218,7 @@ nfsm_mbufuio(struct nfsrv_descript *nd, struct uio
> > > > > *uiop, int siz) }
> > > > >  				mbufcp = NFSMTOD(mp, caddr_t);
> > > > >  				len = mbuf_len(mp);
> > > > > +				KASSERT(len > 0, ("len %d", len));
> > > > >  			}
> > > > >  			xfer = (left > len) ? len : left;
> > > > >  #ifdef notdef
> > > > > @@ -239,6 +240,8 @@ nfsm_mbufuio(struct nfsrv_descript *nd, struct uio
> > > > > *uiop, int siz) uiop->uio_resid -= xfer;
> > > > >  		}
> > > > >  		if (uiop->uio_iov->iov_len <= siz) {
> > > > > +			KASSERT(uiop->uio_iovcnt > 1, ("uio_iovcnt %d",
> > > > > +			    uiop->uio_iovcnt));
> > > > >  			uiop->uio_iovcnt--;
> > > > >  			uiop->uio_iov++;
> > > > >  		} else {
> > > > > 
> > > > > I thought that server have returned too long response, but it seems to
> > > > > be not the case from your data. Still, I think the patch below might be
> > > > > due.
> > > > > 
> > > > > diff --git a/sys/fs/nfsclient/nfs_clrpcops.c
> > > > > b/sys/fs/nfsclient/nfs_clrpcops.c index be0476a..a89b907 100644
> > > > > --- a/sys/fs/nfsclient/nfs_clrpcops.c
> > > > > +++ b/sys/fs/nfsclient/nfs_clrpcops.c
> > > > > @@ -1444,7 +1444,7 @@ nfsrpc_readrpc(vnode_t vp, struct uio *uiop, struct
> > > > > ucred *cred, NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
> > > > >  			eof = fxdr_unsigned(int, *tl);
> > > > >  		}
> > > > > -		NFSM_STRSIZ(retlen, rsize);
> > > > > +		NFSM_STRSIZ(retlen, len);
> > > > >  		error = nfsm_mbufuio(nd, uiop, retlen);
> > > > >  		if (error)
> > > > >  			goto nfsmout;
> > > > 
> > > > I applied your patches and now I get a
> > > > 
> > > > panic: len -4
> > > > cpuid = 1
> > > > KDB: enter: panic
> > > > Dumping 377 out of 6116 MB:..5%..13%..22%..34%..43%..51%..64%..73%..81%..94%
> > > > 
> > > This means that the age driver either produced corrupted mbuf chain,
> > > or filled wrong negative value into the mbuf len field. I am quite
> > > certain that the issue is in the driver.
> > > 
> > > I added the net@ to Cc:, hopefully you could get help there.
> > 
> > And I've cc'd Pyun who has written most of this driver and is likely the one
> > most familiar with its handling of jumbo frames.
> > 
> 
> Try attached one and let me know how it goes.
> Note, I don't have age(4) anymore so it wasn't tested at all.

Sorry, ignore previous patch and use this one(age.diff2) instead.

--8P1HSweYDcXXzwPJ
Content-Type: text/plain; charset=us-ascii
Content-Disposition: attachment; filename="age.diff2"

Index: sys/dev/age/if_age.c
===================================================================
--- sys/dev/age/if_age.c	(revision 245870)
+++ sys/dev/age/if_age.c	(working copy)
@@ -143,6 +143,7 @@ static void age_init_rr_ring(struct age_softc *);
 static void age_init_cmb_block(struct age_softc *);
 static void age_init_smb_block(struct age_softc *);
 static int age_newbuf(struct age_softc *, struct age_rxdesc *);
+static void age_resetbuf(struct age_softc *, int, int);
 static void age_rxvlan(struct age_softc *);
 static void age_rxfilter(struct age_softc *);
 static int sysctl_age_stats(SYSCTL_HANDLER_ARGS);
@@ -2289,9 +2290,7 @@ age_rxeof(struct age_softc *sc, struct rx_rdesc *r
 	nsegs = AGE_RX_NSEGS(index);
 
 	sc->age_cdata.age_rxlen = AGE_RX_BYTES(le32toh(rxrd->len));
-	if ((status & AGE_RRD_ERROR) != 0 &&
-	    (status & (AGE_RRD_CRC | AGE_RRD_CODE | AGE_RRD_DRIBBLE |
-	    AGE_RRD_RUNT | AGE_RRD_OFLOW | AGE_RRD_TRUNC)) != 0) {
+	if ((status & (AGE_RRD_ERROR | AGE_RRD_LENGTH_NOK)) != 0) {
 		/*
 		 * We want to pass the following frames to upper
 		 * layer regardless of error status of Rx return
@@ -2301,9 +2300,12 @@ age_rxeof(struct age_softc *sc, struct rx_rdesc *r
 		 *  o frame length and protocol specific length
 		 *     does not match.
 		 */
-		sc->age_cdata.age_rx_cons += nsegs;
-		sc->age_cdata.age_rx_cons %= AGE_RX_RING_CNT;
-		return;
+		status |= AGE_RRD_IPCSUM_NOK | AGE_RRD_TCP_UDPCSUM_NOK;
+		if ((status & (AGE_RRD_CRC | AGE_RRD_CODE | AGE_RRD_DRIBBLE |
+		    AGE_RRD_RUNT | AGE_RRD_OFLOW | AGE_RRD_TRUNC)) != 0) {
+			age_resetbuf(sc, rx_cons, nsegs);
+			return;
+		}
 	}
 
 	pktlen = 0;
@@ -2316,10 +2318,9 @@ age_rxeof(struct age_softc *sc, struct rx_rdesc *r
 		if (age_newbuf(sc, rxd) != 0) {
 			ifp->if_iqdrops++;
 			/* Reuse Rx buffers. */
-			if (sc->age_cdata.age_rxhead != NULL) {
+			age_resetbuf(sc, rx_cons, nsegs - count);
+			if (sc->age_cdata.age_rxhead != NULL)
 				m_freem(sc->age_cdata.age_rxhead);
-				AGE_RXCHAIN_RESET(sc);
-			}
 			break;
 		}
 
@@ -2383,9 +2384,9 @@ age_rxeof(struct age_softc *sc, struct rx_rdesc *r
 			 */
 			if ((ifp->if_capenable & IFCAP_RXCSUM) != 0 &&
 			    (status & AGE_RRD_IPV4) != 0) {
-				m->m_pkthdr.csum_flags |= CSUM_IP_CHECKED;
 				if ((status & AGE_RRD_IPCSUM_NOK) == 0)
-					m->m_pkthdr.csum_flags |= CSUM_IP_VALID;
+					m->m_pkthdr.csum_flags |=
+					    CSUM_IP_CHECKED | CSUM_IP_VALID;
 				if ((status & (AGE_RRD_TCP | AGE_RRD_UDP)) &&
 				    (status & AGE_RRD_TCP_UDPCSUM_NOK) == 0) {
 					m->m_pkthdr.csum_flags |=
@@ -2411,17 +2412,11 @@ age_rxeof(struct age_softc *sc, struct rx_rdesc *r
 			AGE_UNLOCK(sc);
 			(*ifp->if_input)(ifp, m);
 			AGE_LOCK(sc);
-
-			/* Reset mbuf chains. */
-			AGE_RXCHAIN_RESET(sc);
 		}
 	}
 
-	if (count != nsegs) {
-		sc->age_cdata.age_rx_cons += nsegs;
-		sc->age_cdata.age_rx_cons %= AGE_RX_RING_CNT;
-	} else
-		sc->age_cdata.age_rx_cons = rx_cons;
+	/* Reset mbuf chains. */
+	AGE_RXCHAIN_RESET(sc);
 }
 
 static int
@@ -2460,12 +2455,13 @@ age_rxintr(struct age_softc *sc, int rr_prod, int
 		    (MCLBYTES - ETHER_ALIGN)))
 			break;
 
-		prog++;
 		/* Received a frame. */
 		age_rxeof(sc, rxrd);
 		/* Clear return ring. */
 		rxrd->index = 0;
 		AGE_DESC_INC(rr_cons, AGE_RR_RING_CNT);
+		sc->age_cdata.age_rx_cons += nsegs;
+		sc->age_cdata.age_rx_cons %= AGE_RX_RING_CNT;
 	}
 
 	if (prog > 0) {
@@ -3094,6 +3090,22 @@ age_newbuf(struct age_softc *sc, struct age_rxdesc
 }
 
 static void
+age_resetbuf(struct age_softc *sc, int rx_cons, int count)
+{
+	struct age_rxdesc *rxd;
+	struct rx_desc *desc;
+	int n;
+
+	for (n = 0; n < count; n++) {
+		rxd = &sc->age_cdata.age_rxdesc[rx_cons];
+		desc = rxd->rx_desc;
+		desc->len = htole32(((MCLBYTES - ETHER_ALIGN) &
+		    AGE_RD_LEN_MASK) << AGE_RD_LEN_SHIFT);
+		AGE_DESC_INC(rx_cons, AGE_RX_RING_CNT);
+	}
+}
+
+static void
 age_rxvlan(struct age_softc *sc)
 {
 	struct ifnet *ifp;

--8P1HSweYDcXXzwPJ--



Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?20130125045048.GB1429>