Date: Thu, 17 Feb 2005 15:59:13 -0500 From: Brian Fundakowski Feldman <green@FreeBSD.org> To: ports@FreeBSD.org Subject: portsperf (bsd.port.mk performance bugs fixed) redux Message-ID: <20050217204306.GA1116@green.homeunix.org>
next in thread | raw e-mail | index | archive | help
--WBsA/oQW3eTA3LlM
Content-Type: text/plain; charset=us-ascii
Content-Disposition: inline
Well, guys, it's been a while since the last installment of portsperf.
Without further ado, here's a fix that bring port vulnerability check
time down from almost two minutes to almost instantaneous (on my
machine)! I was always wondering why ports have been so terribly slow
since around some time last year, and a good 2000 pids get recycled
just doing "make extract":
<@green_> now:
<@green_> green# time make extract
<@green_> ===> Extracting for mp3gain-1.3.2
<@green_> => Checksum OK for mp3gain132_src.zip.
<@green_> ===> mp3gain-1.3.2 depends on executable: unzip - found
<@green_> 0.127u 1.132s 0:01.49 83.8% 108+497k 15+0io 8pf+0w
<@green_> then:
<@green_> green# time make extract
<@green_> ===> Extracting for mp3gain-1.3.2
<@green_> => Checksum OK for mp3gain132_src.zip.
<@green_> ===> mp3gain-1.3.2 depends on executable: unzip - found
<@green_> 5.308u 82.765s 1:42.05 86.2% 92+193k 10+0io 12pf+0w
Only problem I know of is that on systems with a good enough version
of the pkg_* tools, an obsolete and broken version (specifically, on
6.0, a version from a year ago) never reports itself as being obsolete
so it can get removed. The pkg_version -T functionality is broken,
then, without a hint to the operator that intervention must occur.
Is that why this change hasn't been made already?
--
Brian Fundakowski Feldman \'[ FreeBSD ]''''''''''\
<> green@FreeBSD.org \ The Power to Serve! \
Opinions expressed are my own. \,,,,,,,,,,,,,,,,,,,,,,\
--WBsA/oQW3eTA3LlM
Content-Type: text/plain; charset=us-ascii
Content-Disposition: attachment; filename=portsperf
Index: bsd.port.mk
===================================================================
RCS file: /usr/ncvs/ports/Mk/bsd.port.mk,v
retrieving revision 1.509
diff -u -r1.509 bsd.port.mk
--- bsd.port.mk 9 Feb 2005 09:07:37 -0000 1.509
+++ bsd.port.mk 17 Feb 2005 20:37:17 -0000
@@ -3009,14 +3009,18 @@
if [ "$$audit_created" -lt "$$audit_expiry" ]; then \
${ECHO_MSG} "===> WARNING: Vulnerability database out of date, checking anyway"; \
fi; \
- vlist=`${_EXTRACT_AUDITFILE} | ${GREP} "${PORTNAME}" | \
- ${AWK} -F\| ' /^[^#]/ { \
- if (!system("${PKG_VERSION} -T \"${PKGNAME}\" \"" $$1 "\"")) \
- print "=> " $$3 ".\n Reference: <" $$2 ">" \
- } \
- '`; \
- if [ -n "$$vlist" ]; then \
+ nvuln=`${_EXTRACT_AUDITFILE} | \
+ ${AWK} -F\| '/^[^#]/ { print $$1; }' | \
+ ${PKG_VERSION} -T "${PKGNAME}" - | \
+ wc -l`; \
+ if [ $$nvuln -ne 0 ]; then \
${ECHO_MSG} "===> ${PKGNAME} has known vulnerabilities:"; \
+ vlist=`${_EXTRACT_AUDITFILE} | ${GREP} "${PORTNAME}" | \
+ ${AWK} -F\| ' /^[^#]/ { \
+ if (!system("${PKG_VERSION} -T \"${PKGNAME}\" \"" $$1 "\"")) \
+ print "=> " $$3 ".\n Reference: <" $$2 ">" \
+ } \
+ '`; \
${ECHO_MSG} "$$vlist"; \
${ECHO_MSG} "=> Please update your ports tree and try again."; \
exit 1; \
@@ -3039,16 +3043,16 @@
file=`${ECHO_CMD} $$_file | ${SED} -E -e 's/:[^:]+$$//'` ; \
select=`${ECHO_CMD} $${_file#$${file}} | ${SED} -e 's/^://' -e 's/,/ /g'` ; \
force_fetch=false; \
- filebasename=`${BASENAME} $$file`; \
+ filebasename=$${file##*/}; \
for afile in ${FORCE_FETCH}; do \
- afile=`${BASENAME} $$afile`; \
+ afile=$${afile##*/}; \
if [ "x$$afile" = "x$$filebasename" ]; then \
force_fetch=true; \
fi; \
done; \
if [ ! -f $$file -a ! -f $$filebasename -o "$$force_fetch" = "true" ]; then \
DIR=${DIST_SUBDIR}; \
- pattern="$${DIR:+$$DIR/}`${ECHO_CMD} $$file | ${SED} -e 's/\./\\\\./g'`"; \
+ pattern="`${ECHO_CMD} $${DIR:+$$DIR/}$$file | ${SED} -e 's:[./]:\\\\&:g'`"; \
if [ -L $$file -o -L $$filebasename ]; then \
${ECHO_MSG} "=> ${_DISTDIR}/$$file is a broken symlink."; \
${ECHO_MSG} "=> Perhaps a filesystem (most likely a CD) isn't mounted?"; \
@@ -3084,7 +3088,7 @@
for site in `eval $$SORTED_MASTER_SITES_CMD_TMP ${_RANDOMIZE_SITES}`; do \
${ECHO_MSG} "=> Attempting to fetch from $${site}."; \
DIR=${DIST_SUBDIR}; \
- CKSIZE=`${GREP} "^SIZE ($${DIR:+$$DIR/}$$file)" ${MD5_FILE} | ${AWK} '{print $$4}'`; \
+ CKSIZE=`${AWK} '/^SIZE/{if ($$2 == "'"($${DIR:+$$DIR/}$$file)"'") {print $$4}}' < ${MD5_FILE}`; \
case $${file} in \
*/*) ${MKDIR} $${file%/*}; \
args="-o $${file} $${site}$${file}";; \
@@ -3106,15 +3110,15 @@
file=`${ECHO_CMD} $$_file | ${SED} -E -e 's/:[^:]+$$//'` ; \
select=`${ECHO_CMD} $${_file#$${file}} | ${SED} -e 's/^://' -e 's/,/ /g'` ; \
force_fetch=false; \
- filebasename=`${BASENAME} $$file`; \
+ filebasename=$${file##*/}; \
for afile in ${FORCE_FETCH}; do \
- afile=`${BASENAME} $$afile`; \
+ afile=$${afile##*/}; \
if [ "x$$afile" = "x$$filebasename" ]; then \
force_fetch=true; \
fi; \
done; \
if [ ! -f $$file -a ! -f $$filebasename -o "$$force_fetch" = "true" ]; then \
- if [ -L $$file -o -L `${BASENAME} $$file` ]; then \
+ if [ -L $$file -o -L $${file##*/} ]; then \
${ECHO_MSG} "=> ${_DISTDIR}/$$file is a broken symlink."; \
${ECHO_MSG} "=> Perhaps a filesystem (most likely a CD) isn't mounted?"; \
${ECHO_MSG} "=> Please correct this problem and try again."; \
@@ -3137,8 +3141,8 @@
for site in `eval $$SORTED_PATCH_SITES_CMD_TMP`; do \
${ECHO_MSG} "=> Attempting to fetch from $${site}."; \
DIR=${DIST_SUBDIR}; \
- pattern="$${DIR:+$$DIR/}`${ECHO_CMD} $$file | ${SED} -e 's/\./\\\\./g'`"; \
- CKSIZE=`${GREP} "^SIZE ($$pattern)" ${MD5_FILE} | ${AWK} '{print $$4}'`; \
+ pattern="`${ECHO_CMD} $${DIR:+$$DIR\\/}$$file | ${SED} -e 's:[./+]:\\\\&:g'`"; \
+ CKSIZE=`${AWK} "/^SIZE \($$pattern\)/"'{print $$4}' < ${MD5_FILE}`; \
case $${file} in \
*/*) ${MKDIR} $${file%/*}; \
args="-o $${file} $${site}$${file}";; \
@@ -4002,7 +4006,7 @@
for _file in ${DISTFILES}; do \
file=`${ECHO_CMD} $$_file | ${SED} -E -e 's/:[^:]+$$//'` ; \
select=`${ECHO_CMD} $${_file#$${file}} | ${SED} -e 's/^://' -e 's/,/ /g'` ; \
- if [ ! -f $$file -a ! -f `${BASENAME} $$file` ]; then \
+ if [ ! -f $$file -a ! -f $${file##*/} ]; then \
if [ ! -z "$$select" ] ; then \
__MASTER_SITES_TMP= ; \
for group in $$select; do \
@@ -4018,7 +4022,7 @@
fi ; \
for site in `eval $$SORTED_MASTER_SITES_CMD_TMP ${_RANDOMIZE_SITES}`; do \
DIR=${DIST_SUBDIR}; \
- CKSIZE=`${GREP} "^SIZE ($${DIR:+$$DIR/}$$file)" ${MD5_FILE} | ${AWK} '{print $$4}'`; \
+ CKSIZE=`${AWK} '/^SIZE/{if ($$2 == "'"($${DIR:+$$DIR/}$$file)"'") {print $$4}}' < ${MD5_FILE}`; \
case $${file} in \
*/*) args="-o $${file} $${site}$${file}";; \
*) args=$${site}$${file};; \
@@ -4034,7 +4038,7 @@
for _file in ${PATCHFILES}; do \
file=`${ECHO_CMD} $$_file | ${SED} -E -e 's/:[^:]+$$//'` ; \
select=`${ECHO_CMD} $${_file#$${file}} | ${SED} -e 's/^://' -e 's/,/ /g'` ; \
- if [ ! -f $$file -a ! -f `${BASENAME} $$file` ]; then \
+ if [ ! -f $$file -a ! -f $${file##*/} ]; then \
if [ ! -z "$$select" ] ; then \
__PATCH_SITES_TMP= ; \
for group in $$select; do \
@@ -4050,7 +4054,7 @@
fi ; \
for site in `eval $$SORTED_PATCH_SITES_CMD_TMP ${_RANDOMIZE_SITES}`; do \
DIR=${DIST_SUBDIR}; \
- CKSIZE=`${GREP} "^SIZE ($${DIR:+$$DIR/}$$file)" ${MD5_FILE} | ${AWK} '{print $$4}'`; \
+ CKSIZE=`${AWK} '/^SIZE/{if ($$2 == "'"($${DIR:+$$DIR/}$$file)"'") {print $$4}}' < ${MD5_FILE}`; \
case $${file} in \
*/*) args="-o $${file} $${site}$${file}";; \
*) args=$${site}$${file};; \
@@ -4101,9 +4105,9 @@
@if [ -f ${MD5_FILE} ]; then \
(cd ${DISTDIR}; OK=""; \
for file in ${_CKSUMFILES}; do \
- pattern="`${ECHO_CMD} $$file | ${SED} -e 's/\./\\\\./g'`"; \
+ pattern="`${ECHO_CMD} $$file | ${SED} -e 's:[./+]:\\\\&:g'`"; \
CKSUM=`${MD5} < $$file`; \
- CKSUM2=`${GREP} "^MD5 ($$pattern)" ${MD5_FILE} | ${AWK} '{print $$4}'`; \
+ CKSUM2=`${AWK} "/^MD5 \($$pattern\)/"'{print $$4}' < ${MD5_FILE}`; \
if [ -z "$$CKSUM2" ]; then \
${ECHO_MSG} "=> No checksum recorded for $$file."; \
OK="false"; \
@@ -4129,8 +4133,8 @@
fi; \
done; \
for file in ${_IGNOREFILES}; do \
- pattern="`${ECHO_CMD} $$file | ${SED} -e 's/\./\\\\./g'`"; \
- CKSUM2=`${GREP} "($$pattern)" ${MD5_FILE} | ${AWK} '{if(NR<2)print $$4}'`; \
+ pattern="`${ECHO_CMD} $$file | ${SED} -e 's:[./+]:\\\\&:g'`"; \
+ CKSUM2=`${AWK} "/\($$pattern\)/"'{print $$4;exit}' < ${MD5_FILE}`; \
if [ "$$CKSUM2" = "" ]; then \
${ECHO_MSG} "=> No checksum recorded for $$file, file is in "'$$'"{IGNOREFILES} list."; \
OK="false"; \
@@ -4216,11 +4220,11 @@
.if defined(${deptype}_DEPENDS)
.if !defined(NO_DEPENDS)
@for i in ${${deptype}_DEPENDS}; do \
- prog=`${ECHO_CMD} $$i | ${SED} -e 's/:.*//'`; \
- dir=`${ECHO_CMD} $$i | ${SED} -e 's/[^:]*://'`; \
+ prog=$${i%%:*}; \
+ dir=$${i##*:}; \
if ${EXPR} "$$dir" : '.*:' > /dev/null; then \
- target=`${ECHO_CMD} $$dir | ${SED} -e 's/.*://'`; \
- dir=`${ECHO_CMD} $$dir | ${SED} -e 's/:.*//'`; \
+ target=$${dir##*:}; \
+ dir=$${dir%%:*}; \
else \
target="${DEPENDS_TARGET}"; \
depends_args="${DEPENDS_ARGS}"; \
@@ -4344,8 +4348,8 @@
.if !defined(NO_DEPENDS)
@for dir in ${DEPENDS}; do \
if ${EXPR} "$$dir" : '.*:' > /dev/null; then \
- target=`${ECHO_CMD} $$dir | ${SED} -e 's/.*://'`; \
- dir=`${ECHO_CMD} $$dir | ${SED} -e 's/:.*//'`; \
+ target=$${dir##*:}; \
+ dir=$${dir%%:*}; \
else \
target="${DEPENDS_TARGET}"; \
depends_args="${DEPENDS_ARGS}"; \
@@ -4374,10 +4378,12 @@
.endif
ALL-DEPENDS-LIST= \
+ hasword() { w=$$1; shift; while [ $$\# -gt 0 ]; do [ "X$$w" = "X$$1" ] && return 0; shift; done; return 1; }; \
checked="${PARENT_CHECKED}"; \
- for dir in $$(${ECHO_CMD} "${EXTRACT_DEPENDS} ${PATCH_DEPENDS} ${FETCH_DEPENDS} ${BUILD_DEPENDS} ${LIB_DEPENDS} ${RUN_DEPENDS}" | ${SED} -e 'y/ /\n/' | ${CUT} -f 2 -d ':') $$(${ECHO_CMD} ${DEPENDS} | ${SED} -e 'y/ /\n/' | ${CUT} -f 1 -d ':'); do \
+ for i in ${EXTRACT_DEPENDS} ${PATCH_DEPENDS} ${FETCH_DEPENDS} ${BUILD_DEPENDS} ${LIB_DEPENDS} ${RUN_DEPENDS}; do j="$${i\#*:}"; dirs="$$dirs $${j%%:*}"; done; for i in ${DEPENDS}; do dirs="$$dirs $${i\#\#*:}"; done; \
+ for dir in $$dirs; do \
if [ -d $$dir ]; then \
- if (${ECHO_CMD} $$checked | ${GREP} -qwv "$$dir"); then \
+ if ! hasword $$dir $$checked; then \
child=$$(cd $$dir; ${MAKE} PARENT_CHECKED="$$checked" all-depends-list); \
for d in $$child; do ${ECHO_CMD} $$d; done; \
${ECHO_CMD} $$dir; \
@@ -4521,6 +4527,7 @@
.endif
PACKAGE-DEPENDS-LIST?= \
+ hasword() { w=$$1; shift; while [ $$\# -gt 0 ]; do [ "X$$w" = "X$$1" ] && return 0; shift; done; return 1; }; \
if [ "${CHILD_DEPENDS}" ]; then \
installed=$$(${PKG_INFO} -qO ${PKGORIGIN} 2>/dev/null || \
${TRUE}); \
@@ -4535,10 +4542,11 @@
done; \
fi; \
checked="${PARENT_CHECKED}"; \
- for dir in $$(${ECHO_CMD} "${LIB_DEPENDS} ${RUN_DEPENDS}" | ${SED} -e 'y/ /\n/' | ${CUT} -f 2 -d ':') $$(${ECHO_CMD} ${DEPENDS} | ${SED} -e 'y/ /\n/' | ${CUT} -f 1 -d ':'); do \
+ for i in ${LIB_DEPENDS} ${RUN_DEPENDS}; do j="$${i\#*:}"; dirs="$$dirs $${j%%:*}"; done; for i in ${DEPENDS}; do dirs="$$dirs $${i\#\#*:}"; done; \
+ for dir in $$dirs; do \
dir=$$(${REALPATH} $$dir); \
if [ -d $$dir ]; then \
- if (${ECHO_CMD} $$checked | ${GREP} -qwv "$$dir"); then \
+ if ! hasword $$dir $$checked; then \
childout=$$(cd $$dir; ${MAKE} CHILD_DEPENDS=yes PARENT_CHECKED="$$checked" package-depends-list); \
set -- $$childout; \
childdir=""; \
--WBsA/oQW3eTA3LlM--
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?20050217204306.GA1116>