[PATCH] test: Better handling of stdout and stderr
Michal Sojka
sojkam1 at fel.cvut.cz
Tue Nov 9 15:40:35 PST 2010
Git-style tests suppress stdout and stderr unless -v is given.
cworth-style tests (created by test_begin_subtest, test_expect_equal)
do not have this behavior so implement it the same.
Additionally, for both test styles, the test-lib.sh is changed so that
the content of suppressed stdout and stderr is shown in case of failed
test.
Finally a test for this functionality is added to basic tests
---
test/basic | 19 ++++++++++++++++++-
test/test-lib.sh | 21 +++++++++++++++------
test/test-verbose | 27 +++++++++++++++++++++++++++
test/test.expected-output/test-verbose-no | 20 ++++++++++++++++++++
test/test.expected-output/test-verbose-yes | 24 ++++++++++++++++++++++++
5 files changed, 104 insertions(+), 7 deletions(-)
create mode 100755 test/test-verbose
create mode 100644 test/test.expected-output/test-verbose-no
create mode 100644 test/test.expected-output/test-verbose-yes
diff --git a/test/basic b/test/basic
index 725c753..091dda0 100755
--- a/test/basic
+++ b/test/basic
@@ -52,9 +52,26 @@ test_expect_code 2 'failure to clean up causes the test to fail' '
# Ensure that all tests are being run
test_begin_subtest 'Ensure that all available tests will be run by notmuch-test'
tests_in_suite=$(grep TESTS= ../notmuch-test | sed -e "s/TESTS=\"\(.*\)\"/\1/" | tr " " "\n" | sort)
-available=$(ls -1 ../ | grep -v -E "^(aggregate-results.sh|Makefile|Makefile.local|notmuch-test|README|test-lib.sh|test-results|tmp.*|valgrind|corpus*|emacs.expected-output|smtp-dummy|smtp-dummy.c)" | sort)
+available=$(ls -1 ../ | grep -v -E "^(aggregate-results.sh|Makefile|Makefile.local|notmuch-test|README|test-lib.sh|test-results|tmp.*|valgrind|corpus*|emacs.expected-output|smtp-dummy|smtp-dummy.c|test-verbose|test.expected-output)" | sort)
test_expect_equal "$tests_in_suite" "$available"
+EXPECTED=../test.expected-output
+suppress_diff_date() {
+ sed -e 's/\(.*\-\-\- test\.4\.\expected\).*/\1/' \
+ -e 's/\(.*\+\+\+ test\.4\.\output\).*/\1/'
+}
+
+test_begin_subtest "Ensure that test output is suppressed unless the test fails"
+output=$(cd ..; ./test-verbose 2>&1 | suppress_diff_date)
+expected=$(cat $EXPECTED/test-verbose-no | suppress_diff_date)
+test_expect_equal "$output" "$expected"
+
+test_begin_subtest "Ensure that -v does not suppress test output"
+output=$(cd ..; ./test-verbose -v 2>&1 | suppress_diff_date)
+expected=$(cat $EXPECTED/test-verbose-yes | suppress_diff_date)
+test_expect_equal "$output" "$expected"
+
+
################################################################
# Test mail store prepared in test-lib.sh
diff --git a/test/test-lib.sh b/test/test-lib.sh
index 8ecc9a0..3f69ab3 100644
--- a/test/test-lib.sh
+++ b/test/test-lib.sh
@@ -165,12 +165,6 @@ fi
echo $(basename "$0"): "Testing ${test_description}"
exec 5>&1
-if test "$verbose" = "t"
-then
- exec 4>&2 3>&1
-else
- exec 4>/dev/null 3>/dev/null
-fi
test_failure=0
test_count=0
@@ -393,6 +387,11 @@ add_email_corpus ()
test_begin_subtest ()
{
test_subtest_name="$1"
+ # Remember stdout and stderr file descriptios and redirect test
+ # output to the previously prepared file descriptors 3 and 4 (see
+ # bellow)
+ if test "$verbose" != "t"; then exec 4>test.output 3>&4; fi
+ exec 6>&1 7>&2 >&3 2>&4
}
# Pass test if two arguments match
@@ -403,6 +402,7 @@ test_begin_subtest ()
# name.
test_expect_equal ()
{
+ exec 1>&6 2>&7 # Restore stdout and stderr
test "$#" = 3 && { prereq=$1; shift; } || prereq=
test "$#" = 2 ||
error "bug in the test script: not 2 or 3 parameters to test_expect_equal"
@@ -498,6 +498,7 @@ test_failure_ () {
echo " $1"
shift
echo "$@" | sed -e 's/^/ /'
+ if test "$verbose" != "t"; then cat test.output; fi
test "$immediate" = "" || { GIT_EXIT_OK=t; exit 1; }
}
@@ -519,6 +520,7 @@ test_debug () {
test_run_ () {
test_cleanup=:
+ if test "$verbose" != "t"; then exec 4>test.output 3>&4; fi
eval >&3 2>&4 "$1"
eval_ret=$?
eval >&3 2>&4 "$test_cleanup"
@@ -908,6 +910,13 @@ EOF
# in subprocesses like git equals our $PWD (for pathname comparisons).
cd -P "$test" || error "Cannot setup test environment"
+if test "$verbose" = "t"
+then
+ exec 4>&2 3>&1
+else
+ exec 4>test.output 3>&4
+fi
+
this_test=${0##*/}
this_test=${this_test%%-*}
for skp in $NOTMUCH_SKIP_TESTS
diff --git a/test/test-verbose b/test/test-verbose
new file mode 100755
index 0000000..ea52f93
--- /dev/null
+++ b/test/test-verbose
@@ -0,0 +1,27 @@
+#!/bin/bash
+
+test_description='the verbosity options of the test framework itself.'
+
+. ./test-lib.sh
+
+test_expect_success 'print something in git-style test and pass' '
+ echo "hello stdout" &&
+ echo "hello stderr" >&2 &&
+ true
+'
+test_expect_success 'print something in git-style test and fail' '
+ echo "hello stdout" &&
+ echo "hello stderr" >&2 &&
+ false
+'
+test_begin_subtest 'print something in cworth-style test and pass'
+echo "hello stdout"
+echo "hello stderr" >&2
+test_expect_equal "a" "a"
+
+test_begin_subtest 'print something in cworth-style test and fail'
+echo "hello stdout"
+echo "hello stderr" >&2
+test_expect_equal "a" "b"
+
+test_done
diff --git a/test/test.expected-output/test-verbose-no b/test/test.expected-output/test-verbose-no
new file mode 100644
index 0000000..04fe71d
--- /dev/null
+++ b/test/test.expected-output/test-verbose-no
@@ -0,0 +1,20 @@
+test-verbose: Testing the verbosity options of the test framework itself.
+ PASS print something in git-style test and pass
+ FAIL print something in git-style test and fail
+
+ echo "hello stdout" &&
+ echo "hello stderr" >&2 &&
+ false
+
+hello stdout
+hello stderr
+ PASS print something in cworth-style test and pass
+ FAIL print something in cworth-style test and fail
+ --- test.4.expected 2010-11-09 23:18:55.597013913 +0000
+ +++ test.4.output 2010-11-09 23:18:55.597013913 +0000
+ @@ -1 +1 @@
+ -b
+ +a
+hello stdout
+hello stderr
+
diff --git a/test/test.expected-output/test-verbose-yes b/test/test.expected-output/test-verbose-yes
new file mode 100644
index 0000000..a84a574
--- /dev/null
+++ b/test/test.expected-output/test-verbose-yes
@@ -0,0 +1,24 @@
+test-verbose: Testing the verbosity options of the test framework itself.
+hello stdout
+hello stderr
+ PASS print something in git-style test and pass
+hello stdout
+hello stderr
+ FAIL print something in git-style test and fail
+
+ echo "hello stdout" &&
+ echo "hello stderr" >&2 &&
+ false
+
+hello stdout
+hello stderr
+ PASS print something in cworth-style test and pass
+hello stdout
+hello stderr
+ FAIL print something in cworth-style test and fail
+ --- test.4.expected 2010-11-09 23:19:05.756552603 +0000
+ +++ test.4.output 2010-11-09 23:19:05.756552603 +0000
+ @@ -1 +1 @@
+ -b
+ +a
+
--
1.7.1
More information about the notmuch
mailing list