Date: Fri, 1 Mar 2002 03:59:09 +0100 From: Christian Weisgerber <naddy@freebsd.org> To: freebsd-ports@freebsd.org Subject: Regression test infrastructure Message-ID: <20020301025909.GA13602@kemoauc.mips.inka.de>
next in thread | raw e-mail | index | archive | help
--OXfL5xGRrasGEqWY
Content-Type: text/plain; charset=us-ascii
Content-Disposition: inline
I would like to add a "regress" target to bsd.port.mk.
Some of the programs in our ports collection ship with regression
tests (ranging from perfunctory to very thorough). Currently there
is no way to run these regression tests from the ports framework.
Having a standard target to run those tests would be very convenient.
This would be a step depending on a successful build, but not called
by default.
OpenBSD already has this in place.
How does it work? You type "make regress". If the port doesn't
have any regression tests (NO_REGRESS=Yes), it will tell you so.
Otherwise, the port will be built (if it hasn't been already) and
"make ${REGRESS_ARGS} ${REGRESS_TARGET}" started in ${REGRESS_WRKSRC}.
Also, REGRESS_DEPENDS can be defined, and the usual {pre-,do-,post-}
targets are available.
Anything that uses the GNU automake infrastructure comes with a
"check" target for running tests, even if no actual tests are hooked
in. Similarly, Perl's MakeMaker provides a "test" target. This
allows us to provide some likely defaults for REGRESS_TARGET based
on *_CONFIGURE.
The implementation is straightforward. It can be mostly cloned
from the "build" target and associated framework. I'm attaching a
first draft version. This closely follows the OpenBSD implementation,
although the actual code is different due to bsd.port.mk divergence.
The patch is relative to Maxim's speed-up patch #3.
To see a simple case in action, apply the bsd.port.mk patches, go
to, say, archivers/gtar (no change required!), and type "make
regress".
--
Christian "naddy" Weisgerber naddy@mips.inka.de
--OXfL5xGRrasGEqWY
Content-Type: text/plain; charset=us-ascii
Content-Disposition: attachment; filename="bsd.port.mk-regress.diff"
--- bsd.port.mk.orig Thu Feb 28 22:58:25 2002
+++ bsd.port.mk Fri Mar 1 03:13:41 2002
@@ -238,6 +238,11 @@
# determine the existence of the dependency is the
# same as FETCH_DEPENDS. If the third field ("target")
# exists, it will be used instead of ${DEPENDS_TARGET}.
+# REGRESS_DEPENDS - A list of "path:dir[:target]" tuples of other ports this
+# package depends on to run regression tests. The test
+# done to determine the existence of the dependency is the
+# same as FETCH_DEPENDS. If the third field ("target")
+# exists, it will be used instead of ${DEPENDS_TARGET}.
# RUN_DEPENDS - A list of "path:dir[:target]" tuples of other ports this
# package depends to run. The test done to determine
# the existence of the dependency is the same as
@@ -344,6 +349,7 @@
# configure - Runs either GNU configure, one or more local configure
# scripts or nothing, depending on what's available.
# build - Actually compile the sources.
+# regress - Run regression tests.
# install - Install the results of a build.
# reinstall - Install the results of a build, ignoring "already installed"
# flag.
@@ -378,6 +384,7 @@
#
# NO_BUILD - Use a dummy (do-nothing) build target.
# NO_INSTALL - Use a dummy (do-nothing) install target.
+# NO_REGRESS - No regression tests available.
#
# Here are some variables used in various stages.
#
@@ -453,6 +460,15 @@
# MAKE_ARGS - Any extra arguments to sub-make in build and install
# stages (default: none).
#
+# For regress:
+#
+# REGRESS_TARGET - Default target for sub-make in regress stage (default:
+# "check" if GNU_CONFIGURE is set, "test" if PERL_CONFIGURE
+# is set, "regress" otherwise).
+# REGRESS_WRKSRC - Directory to run regression tests in (default: ${WRKSRC}).
+# REGRESS_ARGS - Any extra arguments to sub-make in regress stage
+# (default: ${MAKE_ARGS).
+#
# For install:
#
# INSTALL_TARGET - Default target for sub-make in install stage
@@ -823,6 +839,7 @@
PATCH_WRKSRC?= ${WRKSRC}
CONFIGURE_WRKSRC?= ${WRKSRC}
BUILD_WRKSRC?= ${WRKSRC}
+REGRESS_WRKSRC?=${WRKSRC}
INSTALL_WRKSRC?=${WRKSRC}
PLIST_SUB+= OSREL=${OSREL} PREFIX=%D LOCALBASE=${LOCALBASE} X11BASE=${X11BASE}
@@ -1031,6 +1048,7 @@
CONFIGURE_COOKIE?= ${WRKDIR}/.configure_done.${PKGNAME}
INSTALL_COOKIE?= ${WRKDIR}/.install_done.${PKGNAME}
BUILD_COOKIE?= ${WRKDIR}/.build_done.${PKGNAME}
+REGRESS_COOKIE?= ${WRKDIR}/.regress_done.${PKGNAME}
PATCH_COOKIE?= ${WRKDIR}/.patch_done.${PKGNAME}
PACKAGE_COOKIE?= ${WRKDIR}/.package_done.${PKGNAME}
@@ -1624,6 +1642,7 @@
.if defined(PERL_CONFIGURE)
CONFIGURE_ARGS+= CC="${CC}" CCFLAGS="${CFLAGS}" PREFIX="${PREFIX}"
CONFIGURE_SCRIPT?= Makefile.PL
+REGRESS_TARGET?= test
USE_PERL5= yes
.undef HAS_CONFIGURE
.endif
@@ -1635,8 +1654,12 @@
.if defined(GNU_CONFIGURE)
CONFIGURE_ARGS+= --prefix=${PREFIX} ${CONFIGURE_TARGET}
HAS_CONFIGURE= yes
+REGRESS_TARGET?= check
.endif
+REGRESS_TARGET?= regress
+REGRESS_ARGS?= ${MAKE_ARGS}
+
# Passed to most of script invocations
SCRIPTS_ENV+= CURDIR=${MASTERDIR} DISTDIR=${DISTDIR} \
WRKDIR=${WRKDIR} WRKSRC=${WRKSRC} PATCHDIR=${PATCHDIR} \
@@ -1876,6 +1899,8 @@
@${IGNORECMD}
build:
@${IGNORECMD}
+regress:
+ @${IGNORECMD}
install:
@${IGNORECMD}
reinstall:
@@ -1922,7 +1947,8 @@
PATCHDIR=${PATCHDIR} SCRIPTDIR=${SCRIPTDIR} \
FILESDIR=${FILESDIR} PORTSDIR=${PORTSDIR} PREFIX=${PREFIX} \
DEPENDS="${DEPENDS}" BUILD_DEPENDS="${BUILD_DEPENDS}" \
- RUN_DEPENDS="${RUN_DEPENDS}" X11BASE=${X11BASE} \
+ REGRESS_DEPENDS="${REGRESS_DEPENDS}" RUN_DEPENDS="${RUN_DEPENDS}" \
+ X11BASE=${X11BASE} \
${ALL_HOOK}
.endif
@@ -1969,6 +1995,15 @@
@${TOUCH} ${TOUCH_FLAGS} ${BUILD_COOKIE}
.endif
+# Disable regress
+.if defined(NO_REGRESS) && !target(regress)
+regress: build
+.if !defined(IGNORE_SILENT)
+ @${ECHO_MSG} "===> No regression test for ${PKGNAME}."
+.endif
+ @${TOUCH} ${TOUCH_FLAGS} ${REGRESS_COOKIE}
+.endif
+
# Disable install
.if defined(NO_INSTALL) && !target(install)
install: build
@@ -2253,6 +2288,17 @@
.endif
.endif
+# Regress
+
+.if !target(do-regress)
+do-regress:
+.if defined(USE_GMAKE)
+ @(cd ${REGRESS_WRKSRC}; ${SETENV} ${MAKE_ENV} ${GMAKE} ${MAKE_FLAGS} ${MAKEFILE} ${REGRESS_ARGS} ${REGRESS_TARGET})
+.else
+ @(cd ${REGRESS_WRKSRC}; ${SETENV} ${MAKE_ENV} ${MAKE} ${MAKE_FLAGS} ${MAKEFILE} ${REGRESS_ARGS} ${REGRESS_TARGET})
+.endif
+.endif
+
# Install
.if !target(do-install)
@@ -2470,6 +2516,8 @@
post-configure-script
_BUILD_SEQ= configure build-message pre-build pre-build-script do-build \
post-build post-build-script
+_REGRESS_SEQ= build regress-message pre-regress pre-regress-script \
+ do-regress post-regress post-regress-script
_INSTALL_SEQ= build install-message check-categories check-already-installed \
check-umask run-depends lib-depends install-mtree pre-install \
pre-install-script do-install generate-plist post-install \
@@ -2485,7 +2533,7 @@
# Main logick. The loop generates 6 main targets and using cookies
# ensures that those already completed are skipped.
-.for target in extract patch configure build install package
+.for target in extract patch configure build regress install package
.if !target(${target})
.if !defined(USE_SUBMAKE)
@@ -2513,6 +2561,7 @@
.ORDER: ${_PATCH_SEQ}
.ORDER: ${_CONFIGURE_SEQ}
.ORDER: ${_BUILD_SEQ}
+.ORDER: ${_REGRESS_SEQ}
.ORDER: ${_INSTALL_SEQ}
.ORDER: ${_PACKAGE_SEQ}
@@ -2524,6 +2573,8 @@
@${ECHO_MSG} "===> Configuring for ${PKGNAME}"
build-message:
@${ECHO_MSG} "===> Building for ${PKGNAME}"
+regress-message:
+ @${ECHO_MSG} "===> Regression test for ${PKGNAME}"
install-message:
@${ECHO_MSG} "===> Installing for ${PKGNAME}"
package-message:
@@ -2532,7 +2583,7 @@
# Empty pre-* and post-* targets
.for stage in pre post
-.for name in fetch extract patch configure build install package
+.for name in fetch extract patch configure build regress install package
.if !target(${stage}-${name})
${stage}-${name}:
@@ -2882,7 +2933,7 @@
_DEPEND_ALWAYS= 0
.endif
-.for deptype in FETCH BUILD RUN
+.for deptype in FETCH BUILD REGRESS RUN
${deptype:L}-depends:
.if defined(${deptype}_DEPENDS)
.if !defined(NO_DEPENDS)
@@ -3076,6 +3127,18 @@
BUILD-DEPENDS-LIST= \
for dir in $$(${ECHO} "${FETCH_DEPENDS} ${BUILD_DEPENDS} ${LIB_DEPENDS}" | ${TR} '\040' '\012' | ${SED} -e 's/^[^:]*://' -e 's/:.*//' | sort -u) $$(${ECHO} ${DEPENDS} | ${TR} '\040' '\012' | ${SED} -e 's/:.*//' | sort -u); do \
+ if [ -d $$dir ]; then \
+ ${ECHO} $$dir; \
+ else \
+ ${ECHO_MSG} "${PKGNAME}: \"$$dir\" non-existent -- dependency list incomplete" >&2; \
+ fi; \
+ done | sort -u
+
+regress-depends-list:
+ @${REGRESS-DEPENDS-LIST}
+
+REGRESS-DEPENDS-LIST= \
+ for dir in $$(${ECHO} "${FETCH_DEPENDS} ${BUILD_DEPENDS} ${LIB_DEPENDS} ${REGRESS_DEPENDS}" | ${TR} '\040' '\012' | ${SED} -e 's/^[^:]*://' -e 's/:.*//' | sort -u) $$(${ECHO} ${DEPENDS} | ${TR} '\040' '\012' | ${SED} -e 's/:.*//' | sort -u); do \
if [ -d $$dir ]; then \
${ECHO} $$dir; \
else \
--- bsd.port.subdir.mk.orig Thu Feb 28 23:08:54 2002
+++ bsd.port.subdir.mk Fri Mar 1 03:55:20 2002
@@ -32,7 +32,8 @@
# clean-for-cdrom-list, clean-restricted-list,
# configure, deinstall,
# depend, depends, describe, extract, fetch, fetch-list, ignorelist,
-# install, makesum, package, readmes, realinstall, reinstall, tags
+# install, makesum, package, readmes, realinstall, regress,
+# reinstall, tags
#
# search:
# Search for ports using either 'make search key=<keyword>'
@@ -91,6 +92,7 @@
TARGETS+= makesum
TARGETS+= package
TARGETS+= realinstall
+TARGETS+= regress
TARGETS+= reinstall
TARGETS+= tags
--OXfL5xGRrasGEqWY--
To Unsubscribe: send mail to majordomo@FreeBSD.org
with "unsubscribe freebsd-ports" in the body of the message
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?20020301025909.GA13602>