Date: Sun, 13 Nov 2011 03:18:57 +0000 (UTC) From: Alfred Perlstein <alfred@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r227483 - head/usr.sbin/portsnap/portsnap Message-ID: <201111130318.pAD3IvGr074144@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: alfred Date: Sun Nov 13 03:18:57 2011 New Revision: 227483 URL: http://svn.freebsd.org/changeset/base/227483 Log: Utilize shell's IFS instead of forking ~6 processes to handle splitting input files on a '|'. This greatly reduces the time taken to process several databases during the update process. Additionally add some more debug logging. Modified: head/usr.sbin/portsnap/portsnap/portsnap.sh Modified: head/usr.sbin/portsnap/portsnap/portsnap.sh ============================================================================== --- head/usr.sbin/portsnap/portsnap/portsnap.sh Sun Nov 13 03:01:58 2011 (r227482) +++ head/usr.sbin/portsnap/portsnap/portsnap.sh Sun Nov 13 03:18:57 2011 (r227483) @@ -570,14 +570,16 @@ fetch_metadata_sanity() { # Take a list of ${oldhash}|${newhash} and output a list of needed patches fetch_make_patchlist() { - grep -vE "^([0-9a-f]{64})\|\1$" | - while read LINE; do - X=`echo ${LINE} | cut -f 1 -d '|'` - Y=`echo ${LINE} | cut -f 2 -d '|'` - if [ -f "files/${Y}.gz" ]; then continue; fi - if [ ! -f "files/${X}.gz" ]; then continue; fi - echo "${LINE}" + IFS='|' + echo "" 1>${QUIETREDIR} + grep -vE "^([0-9a-f]{64})\|\1$" | + while read X Y; do + printf "Processing: $X $Y ...\r" 1>${QUIETREDIR} + if [ -f "files/${Y}.gz" -o ! -f "files/${X}.gz" ]; then continue; fi + echo "${X}|${Y}" done + echo "" 1>${QUIETREDIR} + IFS= } # Print user-friendly progress statistics @@ -692,9 +694,8 @@ fetch_update() { # Attempt to apply metadata patches echo -n "Applying metadata patches... " - while read LINE; do - X=`echo ${LINE} | cut -f 1 -d '|'` - Y=`echo ${LINE} | cut -f 2 -d '|'` + IFS='|' + while read X Y; do if [ ! -f "${X}-${Y}.gz" ]; then continue; fi gunzip -c < ${X}-${Y}.gz > diff gunzip -c < files/${X}.gz > OLD @@ -707,6 +708,7 @@ fetch_update() { fi rm -f diff OLD NEW ${X}-${Y}.gz ptmp done < patchlist 2>${QUIETREDIR} + IFS= echo "done." # Update metadata without patches @@ -724,16 +726,19 @@ fetch_update() { 2>${QUIETREDIR} while read Y; do + echo -n "Verifying ${Y}... " 1>${QUIETREDIR} if [ `gunzip -c < ${Y}.gz | ${SHA256} -q` = ${Y} ]; then mv ${Y}.gz files/${Y}.gz else echo "metadata is corrupt." return 1 fi + echo "ok." 1>${QUIETREDIR} done < filelist echo "done." # Extract the index + echo -n "Extracting index... " 1>${QUIETREDIR} gunzip -c files/`look INDEX tINDEX.new | cut -f 2 -d '|'`.gz > INDEX.new fetch_index_sanity || return 1 @@ -754,8 +759,10 @@ fetch_update() { fi # Generate a list of wanted ports patches + echo -n "Generating list of wanted patches..." 1>${QUIETREDIR} join -t '|' -o 1.2,2.2 INDEX INDEX.new | fetch_make_patchlist > patchlist + echo " done." 1>${QUIETREDIR} # Attempt to fetch ports patches echo -n "Fetching `wc -l < patchlist | tr -d ' '` " @@ -766,11 +773,18 @@ fetch_update() { echo "done." # Attempt to apply ports patches - echo -n "Applying patches... " - while read LINE; do - X=`echo ${LINE} | cut -f 1 -d '|'` - Y=`echo ${LINE} | cut -f 2 -d '|'` - if [ ! -f "${X}-${Y}" ]; then continue; fi + PATCHCNT=`wc -l patchlist` + echo "Applying patches... " + IFS='|' + I=0 + while read X Y; do + I=$(($I + 1)) + F="${X}-${Y}" + if [ ! -f "${F}" ]; then + printf " Skipping ${F} (${I} of ${PATCHCNT}).\r" + continue; + fi + echo " Processing ${F}..." 1>${QUIETREDIR} gunzip -c < files/${X}.gz > OLD ${BSPATCH} OLD NEW ${X}-${Y} if [ `${SHA256} -q NEW` = ${Y} ]; then @@ -779,6 +793,7 @@ fetch_update() { fi rm -f diff OLD NEW ${X}-${Y} done < patchlist 2>${QUIETREDIR} + IFS= echo "done." # Update ports without patches @@ -795,7 +810,10 @@ fetch_update() { xargs ${XARGST} ${PHTTPGET} ${SERVERNAME} \ 2>${QUIETREDIR} + I=0 while read Y; do + I=$(($I + 1)) + printf " Processing ${Y} (${I} of ${PATCHCNT}).\r" 1>${QUIETREDIR} if [ `gunzip -c < ${Y}.gz | ${SHA256} -q` = ${Y} ]; then mv ${Y}.gz files/${Y}.gz else @@ -877,6 +895,7 @@ extract_metadata() { # Do the actual work involved in "extract" extract_run() { + local IFS='|' mkdir -p ${PORTSDIR} || return 1 if ! @@ -886,7 +905,7 @@ extract_run() { grep -vE "${REFUSE}" ${WORKDIR}/INDEX else cat ${WORKDIR}/INDEX - fi | tr '|' ' ' | while read FILE HASH; do + fi | while read FILE HASH; do echo ${PORTSDIR}/${FILE} if ! [ -r "${WORKDIR}/files/${HASH}.gz" ]; then echo "files/${HASH}.gz not found -- snapshot corrupt." @@ -916,35 +935,8 @@ extract_run() { extract_indices } -# Do the actual work involved in "update" -update_run() { - if ! [ -z "${INDEXONLY}" ]; then - extract_indices >/dev/null || return 1 - return 0 - fi - - if sort ${WORKDIR}/INDEX | - cmp -s ${PORTSDIR}/.portsnap.INDEX -; then - echo "Ports tree is already up to date." - return 0 - fi - -# If we are REFUSEing to touch certain directories, don't remove files -# from those directories (even if they are out of date) - echo -n "Removing old files and directories... " - if ! [ -z "${REFUSE}" ]; then - sort ${WORKDIR}/INDEX | - comm -23 ${PORTSDIR}/.portsnap.INDEX - | cut -f 1 -d '|' | - grep -vE "${REFUSE}" | - lam -s "${PORTSDIR}/" - | - sed -e 's|/$||' | xargs rm -rf - else - sort ${WORKDIR}/INDEX | - comm -23 ${PORTSDIR}/.portsnap.INDEX - | cut -f 1 -d '|' | - lam -s "${PORTSDIR}/" - | - sed -e 's|/$||' | xargs rm -rf - fi - echo "done." +update_run_exract() { + local IFS='|' # Install new files echo "Extracting new files:" @@ -955,9 +947,7 @@ update_run() { sort ${WORKDIR}/INDEX fi | comm -13 ${PORTSDIR}/.portsnap.INDEX - | - while read LINE; do - FILE=`echo ${LINE} | cut -f 1 -d '|'` - HASH=`echo ${LINE} | cut -f 2 -d '|'` + while read FILE HASH; do echo ${PORTSDIR}/${FILE} if ! [ -r "${WORKDIR}/files/${HASH}.gz" ]; then echo "files/${HASH}.gz not found -- snapshot corrupt." @@ -977,7 +967,39 @@ update_run() { done; then return 1 fi +} + +# Do the actual work involved in "update" +update_run() { + if ! [ -z "${INDEXONLY}" ]; then + extract_indices >/dev/null || return 1 + return 0 + fi + + if sort ${WORKDIR}/INDEX | + cmp -s ${PORTSDIR}/.portsnap.INDEX -; then + echo "Ports tree is already up to date." + return 0 + fi + +# If we are REFUSEing to touch certain directories, don't remove files +# from those directories (even if they are out of date) + echo -n "Removing old files and directories... " + if ! [ -z "${REFUSE}" ]; then + sort ${WORKDIR}/INDEX | + comm -23 ${PORTSDIR}/.portsnap.INDEX - | cut -f 1 -d '|' | + grep -vE "${REFUSE}" | + lam -s "${PORTSDIR}/" - | + sed -e 's|/$||' | xargs rm -rf + else + sort ${WORKDIR}/INDEX | + comm -23 ${PORTSDIR}/.portsnap.INDEX - | cut -f 1 -d '|' | + lam -s "${PORTSDIR}/" - | + sed -e 's|/$||' | xargs rm -rf + fi + echo "done." + update_run_exract || return 1 extract_metadata extract_indices }
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201111130318.pAD3IvGr074144>