Date: Sat, 12 Nov 2011 19:25:23 -0800 From: Garrett Cooper <yanegomi@gmail.com> To: Alfred Perlstein <alfred@FreeBSD.org> Cc: svn-src-head@freebsd.org, svn-src-all@freebsd.org, src-committers@freebsd.org Subject: Re: svn commit: r227483 - head/usr.sbin/portsnap/portsnap Message-ID: <32EF12FA-07AC-481C-A407-3C8B578484C4@gmail.com> In-Reply-To: <201111130318.pAD3IvGr074144@svn.freebsd.org> References: <201111130318.pAD3IvGr074144@svn.freebsd.org>
next in thread | previous in thread | raw e-mail | index | archive | help
On Nov 12, 2011, at 7:18 PM, Alfred Perlstein wrote: > Author: alfred > Date: Sun Nov 13 03:18:57 2011 > New Revision: 227483 > URL: http://svn.freebsd.org/changeset/base/227483 >=20 > Log: > Utilize shell's IFS instead of forking ~6 processes to > handle splitting input files on a '|'. This greatly > reduces the time taken to process several databases > during the update process. >=20 > Additionally add some more debug logging. >=20 > Modified: > head/usr.sbin/portsnap/portsnap/portsnap.sh >=20 > Modified: head/usr.sbin/portsnap/portsnap/portsnap.sh > = =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D > --- head/usr.sbin/portsnap/portsnap/portsnap.sh Sun Nov 13 = 03:01:58 2011 (r227482) > +++ head/usr.sbin/portsnap/portsnap/portsnap.sh Sun Nov 13 = 03:18:57 2011 (r227483) > @@ -570,14 +570,16 @@ fetch_metadata_sanity() { >=20 > # Take a list of ${oldhash}|${newhash} and output a list of needed = patches > fetch_make_patchlist() { > - grep -vE "^([0-9a-f]{64})\|\1$" |=20 > - while read LINE; do > - X=3D`echo ${LINE} | cut -f 1 -d '|'` > - Y=3D`echo ${LINE} | cut -f 2 -d '|'` > - if [ -f "files/${Y}.gz" ]; then continue; fi > - if [ ! -f "files/${X}.gz" ]; then continue; fi > - echo "${LINE}" > + IFS=3D'|' > + echo "" 1>${QUIETREDIR} > + grep -vE "^([0-9a-f]{64})\|\1$" | > + while read X Y; do > + printf "Processing: $X $Y ...\r" 1>${QUIETREDIR} > + if [ -f "files/${Y}.gz" -o ! -f "files/${X}.gz" = ]; then continue; fi > + echo "${X}|${Y}" > done > + echo "" 1>${QUIETREDIR} > + IFS=3D > } >=20 > # Print user-friendly progress statistics > @@ -692,9 +694,8 @@ fetch_update() { >=20 > # Attempt to apply metadata patches > echo -n "Applying metadata patches... " > - while read LINE; do > - X=3D`echo ${LINE} | cut -f 1 -d '|'` > - Y=3D`echo ${LINE} | cut -f 2 -d '|'` > + IFS=3D'|' > + while read X Y; do > if [ ! -f "${X}-${Y}.gz" ]; then continue; fi > gunzip -c < ${X}-${Y}.gz > diff > gunzip -c < files/${X}.gz > OLD > @@ -707,6 +708,7 @@ fetch_update() { > fi > rm -f diff OLD NEW ${X}-${Y}.gz ptmp > done < patchlist 2>${QUIETREDIR} > + IFS=3D > echo "done." >=20 > # Update metadata without patches > @@ -724,16 +726,19 @@ fetch_update() { > 2>${QUIETREDIR} >=20 > while read Y; do > + echo -n "Verifying ${Y}... " 1>${QUIETREDIR} > if [ `gunzip -c < ${Y}.gz | ${SHA256} -q` =3D ${Y} ]; = then > mv ${Y}.gz files/${Y}.gz > else > echo "metadata is corrupt." > return 1 > fi > + echo "ok." 1>${QUIETREDIR} > done < filelist > echo "done." >=20 > # Extract the index > + echo -n "Extracting index... " 1>${QUIETREDIR} > gunzip -c files/`look INDEX tINDEX.new | > cut -f 2 -d '|'`.gz > INDEX.new > fetch_index_sanity || return 1 > @@ -754,8 +759,10 @@ fetch_update() { > fi >=20 > # Generate a list of wanted ports patches > + echo -n "Generating list of wanted patches..." 1>${QUIETREDIR} > join -t '|' -o 1.2,2.2 INDEX INDEX.new | > fetch_make_patchlist > patchlist > + echo " done." 1>${QUIETREDIR} >=20 > # Attempt to fetch ports patches > echo -n "Fetching `wc -l < patchlist | tr -d ' '` " > @@ -766,11 +773,18 @@ fetch_update() { > echo "done." >=20 > # Attempt to apply ports patches > - echo -n "Applying patches... " > - while read LINE; do > - X=3D`echo ${LINE} | cut -f 1 -d '|'` > - Y=3D`echo ${LINE} | cut -f 2 -d '|'` > - if [ ! -f "${X}-${Y}" ]; then continue; fi > + PATCHCNT=3D`wc -l patchlist` > + echo "Applying patches... " > + IFS=3D'|' > + I=3D0 > + while read X Y; do > + I=3D$(($I + 1)) > + F=3D"${X}-${Y}" > + if [ ! -f "${F}" ]; then > + printf " Skipping ${F} (${I} of = ${PATCHCNT}).\r" > + continue; > + fi > + echo " Processing ${F}..." 1>${QUIETREDIR} > gunzip -c < files/${X}.gz > OLD > ${BSPATCH} OLD NEW ${X}-${Y} > if [ `${SHA256} -q NEW` =3D ${Y} ]; then > @@ -779,6 +793,7 @@ fetch_update() { > fi > rm -f diff OLD NEW ${X}-${Y} > done < patchlist 2>${QUIETREDIR} > + IFS=3D > echo "done." >=20 > # Update ports without patches > @@ -795,7 +810,10 @@ fetch_update() { > xargs ${XARGST} ${PHTTPGET} ${SERVERNAME} \ > 2>${QUIETREDIR} >=20 > + I=3D0 > while read Y; do > + I=3D$(($I + 1)) > + printf " Processing ${Y} (${I} of ${PATCHCNT}).\r" = 1>${QUIETREDIR} > if [ `gunzip -c < ${Y}.gz | ${SHA256} -q` =3D ${Y} ]; = then > mv ${Y}.gz files/${Y}.gz > else > @@ -877,6 +895,7 @@ extract_metadata() { >=20 > # Do the actual work involved in "extract" > extract_run() { > + local IFS=3D'|' > mkdir -p ${PORTSDIR} || return 1 >=20 > if ! > @@ -886,7 +905,7 @@ extract_run() { > grep -vE "${REFUSE}" ${WORKDIR}/INDEX > else > cat ${WORKDIR}/INDEX > - fi | tr '|' ' ' | while read FILE HASH; do > + fi | while read FILE HASH; do > echo ${PORTSDIR}/${FILE} > if ! [ -r "${WORKDIR}/files/${HASH}.gz" ]; then > echo "files/${HASH}.gz not found -- snapshot = corrupt." > @@ -916,35 +935,8 @@ extract_run() { > extract_indices > } >=20 > -# Do the actual work involved in "update" > -update_run() { > - if ! [ -z "${INDEXONLY}" ]; then > - extract_indices >/dev/null || return 1 > - return 0 > - fi > - > - if sort ${WORKDIR}/INDEX | > - cmp -s ${PORTSDIR}/.portsnap.INDEX -; then > - echo "Ports tree is already up to date." > - return 0 > - fi > - > -# If we are REFUSEing to touch certain directories, don't remove = files > -# from those directories (even if they are out of date) > - echo -n "Removing old files and directories... " > - if ! [ -z "${REFUSE}" ]; then=20 > - sort ${WORKDIR}/INDEX | > - comm -23 ${PORTSDIR}/.portsnap.INDEX - | cut -f 1 -d = '|' | > - grep -vE "${REFUSE}" | > - lam -s "${PORTSDIR}/" - | > - sed -e 's|/$||' | xargs rm -rf > - else > - sort ${WORKDIR}/INDEX | > - comm -23 ${PORTSDIR}/.portsnap.INDEX - | cut -f 1 -d = '|' | > - lam -s "${PORTSDIR}/" - | > - sed -e 's|/$||' | xargs rm -rf > - fi > - echo "done." > +update_run_exract() { Consistent typo :)? > + local IFS=3D'|' >=20 > # Install new files > echo "Extracting new files:" > @@ -955,9 +947,7 @@ update_run() { > sort ${WORKDIR}/INDEX > fi | > comm -13 ${PORTSDIR}/.portsnap.INDEX - | > - while read LINE; do > - FILE=3D`echo ${LINE} | cut -f 1 -d '|'` > - HASH=3D`echo ${LINE} | cut -f 2 -d '|'` > + while read FILE HASH; do > echo ${PORTSDIR}/${FILE} > if ! [ -r "${WORKDIR}/files/${HASH}.gz" ]; then > echo "files/${HASH}.gz not found -- snapshot = corrupt." > @@ -977,7 +967,39 @@ update_run() { > done; then > return 1 > fi > +} > + > +# Do the actual work involved in "update" > +update_run() { > + if ! [ -z "${INDEXONLY}" ]; then > + extract_indices >/dev/null || return 1 > + return 0 > + fi > + > + if sort ${WORKDIR}/INDEX | > + cmp -s ${PORTSDIR}/.portsnap.INDEX -; then > + echo "Ports tree is already up to date." > + return 0 > + fi > + > +# If we are REFUSEing to touch certain directories, don't remove = files > +# from those directories (even if they are out of date) > + echo -n "Removing old files and directories... " > + if ! [ -z "${REFUSE}" ]; then=20 > + sort ${WORKDIR}/INDEX | > + comm -23 ${PORTSDIR}/.portsnap.INDEX - | cut -f 1 -d = '|' | > + grep -vE "${REFUSE}" | > + lam -s "${PORTSDIR}/" - | > + sed -e 's|/$||' | xargs rm -rf > + else > + sort ${WORKDIR}/INDEX | > + comm -23 ${PORTSDIR}/.portsnap.INDEX - | cut -f 1 -d = '|' | > + lam -s "${PORTSDIR}/" - | > + sed -e 's|/$||' | xargs rm -rf > + fi > + echo "done." >=20 > + update_run_exract || return 1 Wouldn't oldIFS=3D$IFS # Blah blah IFS=3D$oldIFS be better? -Garrett=
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?32EF12FA-07AC-481C-A407-3C8B578484C4>