Date: Mon, 20 Jan 2014 08:44:58 -0800 From: Garrett Cooper <yaneurabeya@gmail.com> To: Julio Merino <julio@meroh.net> Cc: freebsd-testing@FreeBSD.org, Jilles Tjoelker <jilles@stack.nl> Subject: [PATCH] convert /bin/sh tests over to ATF Message-ID: <B5290C1B-F262-479C-8D4F-A5D8B3CE5A52@gmail.com>
next in thread | raw e-mail | index | archive | help
[-- Attachment #1 --]
The attached patch converts the /bin/sh tests using the existing framework bin/sh/tests.
All of the tests pass except locale.0, and that’s due to some [currently] unknown issue with my environment — see misc/181151). I could make some of the entries more permanent, but I was trying to make the tests easier to backport by using shell globs.
Also, I added some functionality so the tests expect failures on certain versions of FreeBSD due to missing functionality or conformance modifications (see the .valid_osreldate files).
Thoughts? Comments?
Thanks!
-Garrett
[-- Attachment #2 --]
diff --git a/bin/sh/tests/Makefile b/bin/sh/tests/Makefile
index f6ddb8a..e1424f5 100644
--- a/bin/sh/tests/Makefile
+++ b/bin/sh/tests/Makefile
@@ -4,15 +4,8 @@
TESTSDIR= ${TESTSBASE}/bin/sh
-TAP_TESTS_SH= legacy_test
-TAP_TESTS_SH_SED_legacy_test= -e 's,__SH__,/bin/sh,g'
-# Some tests in here are silently not run when the tests are executed as
-# root. Explicitly tell Kyua to drop privileges.
-#
-# TODO(jmmv): Kyua needs to do this by default, not only when explicitly
-# requested. See https://code.google.com/p/kyua/issues/detail?id=6
-TEST_METADATA.legacy_test+= required_user="unprivileged"
+KYUAFILE= auto
-SUBDIR+= builtins errors execution expansion parameters parser set-e
+ATF_TESTS_SUBDIRS+= builtins errors execution expansion parameters parser set-e
-.include <tap.test.mk>
+.include <atf.test.mk>
diff --git a/bin/sh/tests/Makefile.inc b/bin/sh/tests/Makefile.inc
new file mode 100644
index 0000000..0a79fd8
--- /dev/null
+++ b/bin/sh/tests/Makefile.inc
@@ -0,0 +1,51 @@
+# $FreeBSD$
+
+CLEANFILES= regress.sh
+
+KYUAFILE= auto
+
+SH_TESTS!= cd ${.CURDIR} && find -Es . -regex '.*\.[0-9]+' | sed -e 's,^\./,,g'
+
+SH_TESTS_FILES!= cd ${.CURDIR} && find . \( -type f -and \! -name Makefile\* \) | \
+ sed -e 's,^\./,,g'
+
+TESTSDIR= ${TESTSBASE}/bin/sh/${.CURDIR:T}
+
+FILES+= ${SH_TESTS_FILES}
+FILESDIR= ${TESTSDIR}
+
+regress.sh: ../Makefile.inc ${SH_TESTS_FILES}
+ @echo '' > ${.TARGET}
+ @echo 'OSRELDATE=$$(sysctl -n kern.osreldate)' >> ${.TARGET}
+ @echo ': $${SH=/bin/sh}' >> ${.TARGET}
+ @echo 'export SH' >> ${.TARGET}
+ @echo '' >> ${.TARGET}
+.for test in ${SH_TESTS}
+ @echo 'atf_test_case ${test:S/\//_/g:S/./__/g:S/-/____/g}' >> ${.TARGET}
+ @echo '${test:S/\//_/g:S/./__/g:S/-/____/g}_body() {' >> ${.TARGET}
+ @echo ' cd $$(atf_get_srcdir)' >> ${.TARGET}
+.if exists(${test}.valid_osreldate)
+ @echo -n ' [ '`cat ${.CURDIR}/${test}.valid_osreldate`' ] && ' \
+ >> ${.TARGET}
+ @echo 'atf_expect_fail "expecting failure based on OS version"' \
+ >> ${.TARGET}
+.endif
+ @echo -n ' atf_check -s exit:${test:E} ' >> ${.TARGET}
+.if exists(${test}.stderr)
+ @echo -n '-e "file:$$(atf_get_srcdir)/${test}.stderr" ' >> ${.TARGET}
+.endif
+.if exists(${test}.stdout)
+ @echo -n '-o "file:$$(atf_get_srcdir)/${test}.stdout" ' >> ${.TARGET}
+.endif
+ @echo '$${SH} "./${test}"' >> ${.TARGET}
+ @echo '}' >> ${.TARGET}
+ @echo '' >> ${.TARGET}
+.endfor
+ @echo '' >> ${.TARGET}
+ @echo 'atf_init_test_cases() {' >> ${.TARGET}
+.for test in ${SH_TESTS}
+ @echo ' atf_add_test_case ${test:S/\//_/g:S/./__/g:S/-/____/g}' >> ${.TARGET}
+.endfor
+ @echo '}' >> ${.TARGET}
+
+ATF_TESTS_SH+= regress
diff --git a/bin/sh/tests/builtins/Makefile b/bin/sh/tests/builtins/Makefile
index 3b1d24b..343219d 100644
--- a/bin/sh/tests/builtins/Makefile
+++ b/bin/sh/tests/builtins/Makefile
@@ -2,146 +2,4 @@
.include <bsd.own.mk>
-FILESDIR= ${TESTSBASE}/bin/sh/builtins
-KYUAFILE= no
-
-FILES= alias.0 alias.0.stdout
-FILES+= alias.1 alias.1.stderr
-FILES+= alias3.0 alias3.0.stdout
-FILES+= alias4.0
-FILES+= break1.0
-FILES+= break2.0 break2.0.stdout
-FILES+= break3.0
-FILES+= break4.4
-FILES+= break5.4
-FILES+= builtin1.0
-FILES+= case1.0
-FILES+= case2.0
-FILES+= case3.0
-FILES+= case4.0
-FILES+= case5.0
-FILES+= case6.0
-FILES+= case7.0
-FILES+= case8.0
-FILES+= case9.0
-FILES+= case10.0
-FILES+= case11.0
-FILES+= case12.0
-FILES+= case13.0
-FILES+= case14.0
-FILES+= case15.0
-FILES+= case16.0
-FILES+= case17.0
-FILES+= case18.0
-FILES+= case19.0
-FILES+= cd1.0
-FILES+= cd2.0
-FILES+= cd3.0
-FILES+= cd4.0
-FILES+= cd5.0
-FILES+= cd6.0
-FILES+= cd7.0
-FILES+= cd8.0
-FILES+= command1.0
-FILES+= command2.0
-FILES+= command3.0
-FILES+= command3.0.stdout
-FILES+= command4.0
-FILES+= command5.0
-FILES+= command5.0.stdout
-FILES+= command6.0
-FILES+= command6.0.stdout
-FILES+= command7.0
-FILES+= command8.0
-FILES+= command9.0
-FILES+= command10.0
-FILES+= command11.0
-FILES+= command12.0
-FILES+= dot1.0
-FILES+= dot2.0
-FILES+= dot3.0
-FILES+= dot4.0
-FILES+= eval1.0
-FILES+= eval2.0
-FILES+= eval3.0
-FILES+= eval4.0
-FILES+= eval5.0
-FILES+= eval6.0
-FILES+= exec1.0
-FILES+= exec2.0
-FILES+= exit1.0
-FILES+= exit2.8
-FILES+= exit3.0
-FILES+= export1.0
-FILES+= fc1.0
-FILES+= fc2.0
-FILES+= for1.0
-FILES+= for2.0
-FILES+= for3.0
-FILES+= getopts1.0 getopts1.0.stdout
-FILES+= getopts2.0 getopts2.0.stdout
-FILES+= hash1.0 hash1.0.stdout
-FILES+= hash2.0 hash2.0.stdout
-FILES+= hash3.0 hash3.0.stdout
-FILES+= hash4.0
-FILES+= jobid1.0
-FILES+= jobid2.0
-FILES+= lineno.0 lineno.0.stdout
-FILES+= local1.0
-FILES+= local2.0
-FILES+= local3.0
-FILES+= local4.0
-FILES+= locale1.0
-FILES+= printf1.0
-FILES+= printf2.0
-FILES+= printf3.0
-FILES+= printf4.0
-FILES+= read1.0 read1.0.stdout
-FILES+= read2.0
-FILES+= read3.0 read3.0.stdout
-FILES+= read4.0 read4.0.stdout
-FILES+= read5.0
-FILES+= read6.0
-FILES+= read7.0
-FILES+= return1.0
-FILES+= return2.1
-FILES+= return3.1
-FILES+= return4.0
-FILES+= return5.0
-FILES+= return6.4
-FILES+= return7.4
-FILES+= return8.0
-FILES+= set1.0
-FILES+= set2.0
-FILES+= trap1.0
-FILES+= trap10.0
-FILES+= trap11.0
-FILES+= trap12.0
-FILES+= trap13.0
-FILES+= trap14.0
-FILES+= trap2.0
-FILES+= trap3.0
-FILES+= trap4.0
-FILES+= trap5.0
-FILES+= trap6.0
-FILES+= trap7.0
-FILES+= trap8.0
-FILES+= trap9.0
-FILES+= type1.0 type1.0.stderr
-FILES+= type2.0
-FILES+= type3.0
-FILES+= unalias.0
-FILES+= var-assign.0
-FILES+= var-assign2.0
-FILES+= wait1.0
-FILES+= wait2.0
-FILES+= wait3.0
-FILES+= wait4.0
-FILES+= wait5.0
-FILES+= wait6.0
-FILES+= wait7.0
-FILES+= wait8.0
-FILES+= wait9.127
-FILES+= wait10.0
-
-.include <bsd.test.mk>
+.include <atf.test.mk>
diff --git a/bin/sh/tests/builtins/alias.1.valid_osreldate b/bin/sh/tests/builtins/alias.1.valid_osreldate
new file mode 100644
index 0000000..96bc086
--- /dev/null
+++ b/bin/sh/tests/builtins/alias.1.valid_osreldate
@@ -0,0 +1 @@
+\( ${OSRELDATE} -gt 802506 -a ${OSRELDATE} -lt 900000 \)
diff --git a/bin/sh/tests/errors/Makefile b/bin/sh/tests/errors/Makefile
index 9f8b0f2..f0ba343 100644
--- a/bin/sh/tests/errors/Makefile
+++ b/bin/sh/tests/errors/Makefile
@@ -2,9 +2,6 @@
.include <bsd.own.mk>
-FILESDIR= ${TESTSBASE}/bin/sh/errors
-KYUAFILE= no
-
FILES= assignment-error1.0
FILES+= assignment-error2.0
FILES+= backquote-error1.0
@@ -27,4 +24,4 @@ FILES+= redirection-error6.0
FILES+= redirection-error7.0
FILES+= write-error1.0
-.include <bsd.test.mk>
+.include <atf.test.mk>
diff --git a/bin/sh/tests/errors/bad-parm-exp2.2.stderr b/bin/sh/tests/errors/bad-parm-exp2.2.stderr
index d027a5a..1135af0 100644
--- a/bin/sh/tests/errors/bad-parm-exp2.2.stderr
+++ b/bin/sh/tests/errors/bad-parm-exp2.2.stderr
@@ -1 +1 @@
-./errors/bad-parm-exp2.2: ${}: Bad substitution
+./bad-parm-exp2.2: ${}: Bad substitution
diff --git a/bin/sh/tests/errors/bad-parm-exp3.2.stderr b/bin/sh/tests/errors/bad-parm-exp3.2.stderr
index ef40251..b28fdff 100644
--- a/bin/sh/tests/errors/bad-parm-exp3.2.stderr
+++ b/bin/sh/tests/errors/bad-parm-exp3.2.stderr
@@ -1 +1 @@
-./errors/bad-parm-exp3.2: ${foo/}: Bad substitution
+./bad-parm-exp3.2: ${foo/}: Bad substitution
diff --git a/bin/sh/tests/errors/bad-parm-exp4.2.stderr b/bin/sh/tests/errors/bad-parm-exp4.2.stderr
index 89bd80f..1487b61 100644
--- a/bin/sh/tests/errors/bad-parm-exp4.2.stderr
+++ b/bin/sh/tests/errors/bad-parm-exp4.2.stderr
@@ -1 +1 @@
-./errors/bad-parm-exp4.2: ${foo:@...}: Bad substitution
+./bad-parm-exp4.2: ${foo:@...}: Bad substitution
diff --git a/bin/sh/tests/errors/bad-parm-exp5.2.stderr b/bin/sh/tests/errors/bad-parm-exp5.2.stderr
index 89b1997..2138c2e 100644
--- a/bin/sh/tests/errors/bad-parm-exp5.2.stderr
+++ b/bin/sh/tests/errors/bad-parm-exp5.2.stderr
@@ -1 +1 @@
-./errors/bad-parm-exp5.2: ${/}: Bad substitution
+./bad-parm-exp5.2: ${/}: Bad substitution
diff --git a/bin/sh/tests/errors/bad-parm-exp6.2.stderr b/bin/sh/tests/errors/bad-parm-exp6.2.stderr
index dbf14b5..ba3b671 100644
--- a/bin/sh/tests/errors/bad-parm-exp6.2.stderr
+++ b/bin/sh/tests/errors/bad-parm-exp6.2.stderr
@@ -1 +1 @@
-./errors/bad-parm-exp6.2: ${foo...}: Bad substitution
+./bad-parm-exp6.2: ${foo...}: Bad substitution
diff --git a/bin/sh/tests/execution/Makefile b/bin/sh/tests/execution/Makefile
index 302d0d8..343219d 100644
--- a/bin/sh/tests/execution/Makefile
+++ b/bin/sh/tests/execution/Makefile
@@ -2,52 +2,4 @@
.include <bsd.own.mk>
-FILESDIR= ${TESTSBASE}/bin/sh/execution
-KYUAFILE= no
-
-FILES= bg1.0
-FILES+= bg2.0
-FILES+= bg3.0
-FILES+= bg4.0
-FILES+= bg5.0
-FILES+= bg6.0 bg6.0.stdout
-FILES+= bg7.0
-FILES+= bg8.0
-FILES+= bg9.0
-FILES+= bg10.0 bg10.0.stdout
-FILES+= fork1.0
-FILES+= fork2.0
-FILES+= fork3.0
-FILES+= func1.0
-FILES+= func2.0
-FILES+= func3.0
-FILES+= hash1.0
-FILES+= int-cmd1.0
-FILES+= killed1.0
-FILES+= killed2.0
-FILES+= not1.0
-FILES+= not2.0
-FILES+= path1.0
-FILES+= redir1.0
-FILES+= redir2.0
-FILES+= redir3.0
-FILES+= redir4.0
-FILES+= redir5.0
-FILES+= redir6.0
-FILES+= redir7.0
-FILES+= set-n1.0
-FILES+= set-n2.0
-FILES+= set-n3.0
-FILES+= set-n4.0
-FILES+= set-x1.0
-FILES+= set-x2.0
-FILES+= set-x3.0
-FILES+= shellproc1.0
-FILES+= subshell1.0 subshell1.0.stdout
-FILES+= subshell2.0
-FILES+= subshell3.0
-FILES+= subshell4.0
-FILES+= unknown1.0
-FILES+= var-assign1.0
-
-.include <bsd.test.mk>
+.include <atf.test.mk>
diff --git a/bin/sh/tests/expansion/Makefile b/bin/sh/tests/expansion/Makefile
index bd24319..343219d 100644
--- a/bin/sh/tests/expansion/Makefile
+++ b/bin/sh/tests/expansion/Makefile
@@ -2,85 +2,4 @@
.include <bsd.own.mk>
-FILESDIR= ${TESTSBASE}/bin/sh/expansion
-KYUAFILE= no
-
-FILES= arith1.0
-FILES+= arith2.0
-FILES+= arith3.0
-FILES+= arith4.0
-FILES+= arith5.0
-FILES+= arith6.0
-FILES+= arith7.0
-FILES+= arith8.0
-FILES+= arith9.0
-FILES+= arith10.0
-FILES+= arith11.0
-FILES+= arith12.0
-FILES+= arith13.0
-FILES+= assign1.0
-FILES+= cmdsubst1.0
-FILES+= cmdsubst2.0
-FILES+= cmdsubst3.0
-FILES+= cmdsubst4.0
-FILES+= cmdsubst5.0
-FILES+= cmdsubst6.0
-FILES+= cmdsubst7.0
-FILES+= cmdsubst8.0
-FILES+= cmdsubst9.0
-FILES+= cmdsubst10.0
-FILES+= cmdsubst11.0
-FILES+= cmdsubst12.0
-FILES+= cmdsubst13.0
-FILES+= cmdsubst14.0
-FILES+= cmdsubst15.0
-FILES+= cmdsubst16.0
-FILES+= cmdsubst17.0
-FILES+= export1.0
-FILES+= export2.0
-FILES+= export3.0
-FILES+= heredoc1.0
-FILES+= heredoc2.0
-FILES+= ifs1.0
-FILES+= ifs2.0
-FILES+= ifs3.0
-FILES+= ifs4.0
-FILES+= length1.0
-FILES+= length2.0
-FILES+= length3.0
-FILES+= length4.0
-FILES+= length5.0
-FILES+= length6.0
-FILES+= length7.0
-FILES+= length8.0
-FILES+= local1.0
-FILES+= local2.0
-FILES+= pathname1.0
-FILES+= pathname2.0
-FILES+= pathname3.0
-FILES+= pathname4.0
-FILES+= plus-minus1.0
-FILES+= plus-minus2.0
-FILES+= plus-minus3.0
-FILES+= plus-minus4.0
-FILES+= plus-minus5.0
-FILES+= plus-minus6.0
-FILES+= plus-minus7.0
-FILES+= plus-minus8.0
-FILES+= question1.0
-FILES+= readonly1.0
-FILES+= set-u1.0
-FILES+= set-u2.0
-FILES+= set-u3.0
-FILES+= tilde1.0
-FILES+= tilde2.0
-FILES+= trim1.0
-FILES+= trim2.0
-FILES+= trim3.0
-FILES+= trim4.0
-FILES+= trim5.0
-FILES+= trim6.0
-FILES+= trim7.0
-FILES+= trim8.0
-
-.include <bsd.test.mk>
+.include <atf.test.mk>
diff --git a/bin/sh/tests/legacy_test.sh b/bin/sh/tests/legacy_test.sh
deleted file mode 100644
index d43f5dd..0000000
--- a/bin/sh/tests/legacy_test.sh
+++ /dev/null
@@ -1,46 +0,0 @@
-# $FreeBSD$
-
-: ${SH:="__SH__"}
-export SH
-
-# TODO(jmmv): The Kyua TAP interface should be passing us the value of
-# "srcdir" as an environment variable, just as it does with the ATF
-# interface in the form of a configuration variable. For now, just try
-# to guess this.
-: ${TESTS_DATA:=$(dirname ${0})}
-
-COUNTER=1
-
-do_test() {
- c=${COUNTER}
- COUNTER=$((COUNTER+1))
- ${SH} $1 > tmp.stdout 2> tmp.stderr
- if [ $? -ne $2 ]; then
- echo "not ok ${c} - ${1} # wrong exit status"
- rm tmp.stdout tmp.stderr
- return
- fi
- sed -I '' -e "s|^${TESTS_DATA}|.|" tmp.stderr
- for i in stdout stderr; do
- if [ -f ${1}.${i} ]; then
- if ! cmp -s tmp.${i} ${1}.${i}; then
- echo "not ok ${c} - ${1} # wrong output on ${i}"
- rm tmp.stdout tmp.stderr
- return
- fi
- elif [ -s tmp.${i} ]; then
- echo "not ok ${c} - ${1} # wrong output on ${i}"
- rm tmp.stdout tmp.stderr
- return
- fi
- done
- echo "ok ${c} - ${1}"
- rm tmp.stdout tmp.stderr
-}
-
-TESTS=$(find -Es ${TESTS_DATA} -regex ".*\.[0-9]+")
-printf "1..%d\n" $(echo ${TESTS} | wc -w)
-
-for i in ${TESTS} ; do
- do_test ${i} ${i##*.}
-done
diff --git a/bin/sh/tests/parameters/Makefile b/bin/sh/tests/parameters/Makefile
index fafc059..343219d 100644
--- a/bin/sh/tests/parameters/Makefile
+++ b/bin/sh/tests/parameters/Makefile
@@ -2,18 +2,4 @@
.include <bsd.own.mk>
-FILESDIR= ${TESTSBASE}/bin/sh/parameters
-KYUAFILE= no
-
-FILES= env1.0
-FILES+= exitstatus1.0
-FILES+= mail1.0
-FILES+= mail2.0
-FILES+= optind1.0
-FILES+= optind2.0
-FILES+= positional1.0
-FILES+= positional2.0
-FILES+= pwd1.0
-FILES+= pwd2.0
-
-.include <bsd.test.mk>
+.include <atf.test.mk>
diff --git a/bin/sh/tests/parser/Makefile b/bin/sh/tests/parser/Makefile
index 71bd370..343219d 100644
--- a/bin/sh/tests/parser/Makefile
+++ b/bin/sh/tests/parser/Makefile
@@ -2,58 +2,4 @@
.include <bsd.own.mk>
-FILESDIR= ${TESTSBASE}/bin/sh/parser
-KYUAFILE= no
-
-FILES= alias1.0
-FILES+= alias2.0
-FILES+= alias3.0
-FILES+= alias4.0
-FILES+= alias5.0
-FILES+= alias6.0
-FILES+= alias7.0
-FILES+= alias8.0
-FILES+= alias9.0
-FILES+= alias10.0
-FILES+= and-pipe-not.0
-FILES+= case1.0
-FILES+= case2.0
-FILES+= dollar-quote1.0
-FILES+= dollar-quote2.0
-FILES+= dollar-quote3.0
-FILES+= dollar-quote4.0
-FILES+= dollar-quote5.0
-FILES+= dollar-quote6.0
-FILES+= dollar-quote7.0
-FILES+= dollar-quote8.0
-FILES+= dollar-quote9.0
-FILES+= dollar-quote10.0
-FILES+= dollar-quote11.0
-FILES+= empty-braces1.0
-FILES+= empty-cmd1.0
-FILES+= for1.0
-FILES+= for2.0
-FILES+= func1.0
-FILES+= func2.0
-FILES+= func3.0
-FILES+= heredoc1.0
-FILES+= heredoc2.0
-FILES+= heredoc3.0
-FILES+= heredoc4.0
-FILES+= heredoc5.0
-FILES+= heredoc6.0
-FILES+= heredoc7.0
-FILES+= heredoc8.0
-FILES+= heredoc9.0
-FILES+= heredoc10.0
-FILES+= heredoc11.0
-FILES+= no-space1.0
-FILES+= no-space2.0
-FILES+= only-redir1.0
-FILES+= only-redir2.0
-FILES+= only-redir3.0
-FILES+= only-redir4.0
-FILES+= pipe-not1.0
-FILES+= var-assign1.0
-
-.include <bsd.test.mk>
+.include <atf.test.mk>
diff --git a/bin/sh/tests/set-e/Makefile b/bin/sh/tests/set-e/Makefile
index 55d7917..343219d 100644
--- a/bin/sh/tests/set-e/Makefile
+++ b/bin/sh/tests/set-e/Makefile
@@ -2,43 +2,4 @@
.include <bsd.own.mk>
-FILESDIR= ${TESTSBASE}/bin/sh/set-e
-KYUAFILE= no
-
-FILES= and1.0
-FILES+= and2.1
-FILES+= and3.0
-FILES+= and4.0
-FILES+= background1.0
-FILES+= cmd1.0
-FILES+= cmd2.1
-FILES+= elif1.0
-FILES+= elif2.0
-FILES+= eval1.0
-FILES+= eval2.1
-FILES+= for1.0
-FILES+= func1.0
-FILES+= func2.1
-FILES+= if1.0
-FILES+= if2.0
-FILES+= if3.0
-FILES+= not1.0
-FILES+= not2.0
-FILES+= or1.0
-FILES+= or2.0
-FILES+= or3.1
-FILES+= pipe1.1
-FILES+= pipe2.0
-FILES+= return1.0
-FILES+= semi1.1
-FILES+= semi2.1
-FILES+= subshell1.0
-FILES+= subshell2.1
-FILES+= until1.0
-FILES+= until2.0
-FILES+= until3.0
-FILES+= while1.0
-FILES+= while2.0
-FILES+= while3.0
-
-.include <bsd.test.mk>
+.include <atf.test.mk>
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?B5290C1B-F262-479C-8D4F-A5D8B3CE5A52>