[PATCH 1/5] test: Better handling of stdout and stderr
Michal Sojka
sojkam1 at fel.cvut.cz
Sun Nov 14 13:54:28 PST 2010
Git-style tests (test_expect_success etc.) suppress stdout and stderr
unless -v is given. Notmuch-style tests (created by test_begin_subtest
and test_expect_equal) do not have this behavior so implement it the
same.
Additionally, for both test styles, the test-lib.sh is changed so that
the content of suppressed stdout and stderr is shown in case of failed
test.
Finally a test for this functionality is added to basic tests.
---
test/basic | 21 ++++++++++++++++++++-
test/test-lib.sh | 21 +++++++++++++++------
test/test-verbose | 27 +++++++++++++++++++++++++++
test/test.expected-output/test-verbose-no | 20 ++++++++++++++++++++
test/test.expected-output/test-verbose-yes | 24 ++++++++++++++++++++++++
5 files changed, 106 insertions(+), 7 deletions(-)
create mode 100755 test/test-verbose
create mode 100644 test/test.expected-output/test-verbose-no
create mode 100644 test/test.expected-output/test-verbose-yes
diff --git a/test/basic b/test/basic
index 725c753..309779c 100755
--- a/test/basic
+++ b/test/basic
@@ -52,9 +52,28 @@ test_expect_code 2 'failure to clean up causes the test to fail' '
# Ensure that all tests are being run
test_begin_subtest 'Ensure that all available tests will be run by notmuch-test'
tests_in_suite=$(grep TESTS= ../notmuch-test | sed -e "s/TESTS=\"\(.*\)\"/\1/" | tr " " "\n" | sort)
-available=$(ls -1 ../ | grep -v -E "^(aggregate-results.sh|Makefile|Makefile.local|notmuch-test|README|test-lib.sh|test-results|tmp.*|valgrind|corpus*|emacs.expected-output|smtp-dummy|smtp-dummy.c)" | sort)
+available=$(ls -1 ../ | grep -v -E "^(aggregate-results.sh|Makefile|Makefile.local|notmuch-test|README|test-lib.sh|test-results|tmp.*|valgrind|corpus*|emacs.expected-output|smtp-dummy|smtp-dummy.c|test-verbose|test.expected-output)" | sort)
test_expect_equal "$tests_in_suite" "$available"
+EXPECTED=../test.expected-output
+suppress_diff_date() {
+ sed -e 's/\(.*\-\-\- test-verbose\.4\.\expected\).*/\1/' \
+ -e 's/\(.*\+\+\+ test-verbose\.4\.\output\).*/\1/'
+}
+
+test_begin_subtest "Ensure that test output is suppressed unless the test fails"
+output=$(cd ..; ./test-verbose 2>&1 | suppress_diff_date)
+expected=$(cat $EXPECTED/test-verbose-no | suppress_diff_date)
+test_expect_equal "$output" "$expected"
+
+test_begin_subtest "Ensure that -v does not suppress test output"
+output=$(cd ..; ./test-verbose -v 2>&1 | suppress_diff_date)
+expected=$(cat $EXPECTED/test-verbose-yes | suppress_diff_date)
+# Do not include the results of test-verbose in totals
+rm $TEST_DIRECTORY/test-results/test-verbose-*
+test_expect_equal "$output" "$expected"
+
+
################################################################
# Test mail store prepared in test-lib.sh
diff --git a/test/test-lib.sh b/test/test-lib.sh
index 418eaa7..68c9cf8 100644
--- a/test/test-lib.sh
+++ b/test/test-lib.sh
@@ -165,12 +165,6 @@ fi
echo $(basename "$0"): "Testing ${test_description}"
exec 5>&1
-if test "$verbose" = "t"
-then
- exec 4>&2 3>&1
-else
- exec 4>/dev/null 3>/dev/null
-fi
test_failure=0
test_count=0
@@ -403,6 +397,11 @@ add_email_corpus ()
test_begin_subtest ()
{
test_subtest_name="$1"
+ # Remember stdout and stderr file descriptios and redirect test
+ # output to the previously prepared file descriptors 3 and 4 (see
+ # bellow)
+ if test "$verbose" != "t"; then exec 4>test.output 3>&4; fi
+ exec 6>&1 7>&2 >&3 2>&4
}
# Pass test if two arguments match
@@ -413,6 +412,7 @@ test_begin_subtest ()
# name.
test_expect_equal ()
{
+ exec 1>&6 2>&7 # Restore stdout and stderr
test "$#" = 3 && { prereq=$1; shift; } || prereq=
test "$#" = 2 ||
error "bug in the test script: not 2 or 3 parameters to test_expect_equal"
@@ -508,6 +508,7 @@ test_failure_ () {
echo " $1"
shift
echo "$@" | sed -e 's/^/ /'
+ if test "$verbose" != "t"; then cat test.output; fi
test "$immediate" = "" || { GIT_EXIT_OK=t; exit 1; }
}
@@ -529,6 +530,7 @@ test_debug () {
test_run_ () {
test_cleanup=:
+ if test "$verbose" != "t"; then exec 4>test.output 3>&4; fi
eval >&3 2>&4 "$1"
eval_ret=$?
eval >&3 2>&4 "$test_cleanup"
@@ -918,6 +920,13 @@ EOF
# in subprocesses like git equals our $PWD (for pathname comparisons).
cd -P "$test" || error "Cannot setup test environment"
+if test "$verbose" = "t"
+then
+ exec 4>&2 3>&1
+else
+ exec 4>test.output 3>&4
+fi
+
this_test=${0##*/}
for skp in $NOTMUCH_SKIP_TESTS
do
diff --git a/test/test-verbose b/test/test-verbose
new file mode 100755
index 0000000..f29a9c7
--- /dev/null
+++ b/test/test-verbose
@@ -0,0 +1,27 @@
+#!/bin/bash
+
+test_description='the verbosity options of the test framework itself.'
+
+. ./test-lib.sh
+
+test_expect_success 'print something in test_expect_success and pass' '
+ echo "hello stdout" &&
+ echo "hello stderr" >&2 &&
+ true
+'
+test_expect_success 'print something in test_expect_success and fail' '
+ echo "hello stdout" &&
+ echo "hello stderr" >&2 &&
+ false
+'
+test_begin_subtest 'print something between test_begin_subtest and test_expect_equal and pass'
+echo "hello stdout"
+echo "hello stderr" >&2
+test_expect_equal "a" "a"
+
+test_begin_subtest 'print something test_begin_subtest and test_expect_equal and fail'
+echo "hello stdout"
+echo "hello stderr" >&2
+test_expect_equal "a" "b"
+
+test_done
diff --git a/test/test.expected-output/test-verbose-no b/test/test.expected-output/test-verbose-no
new file mode 100644
index 0000000..0bca754
--- /dev/null
+++ b/test/test.expected-output/test-verbose-no
@@ -0,0 +1,20 @@
+test-verbose: Testing the verbosity options of the test framework itself.
+ PASS print something in test_expect_success and pass
+ FAIL print something in test_expect_success and fail
+
+ echo "hello stdout" &&
+ echo "hello stderr" >&2 &&
+ false
+
+hello stdout
+hello stderr
+ PASS print something between test_begin_subtest and test_expect_equal and pass
+ FAIL print something test_begin_subtest and test_expect_equal and fail
+ --- test-verbose.4.expected 2010-11-14 21:41:12.738189710 +0000
+ +++ test-verbose.4.output 2010-11-14 21:41:12.738189710 +0000
+ @@ -1 +1 @@
+ -b
+ +a
+hello stdout
+hello stderr
+
diff --git a/test/test.expected-output/test-verbose-yes b/test/test.expected-output/test-verbose-yes
new file mode 100644
index 0000000..ebe5187
--- /dev/null
+++ b/test/test.expected-output/test-verbose-yes
@@ -0,0 +1,24 @@
+test-verbose: Testing the verbosity options of the test framework itself.
+hello stdout
+hello stderr
+ PASS print something in test_expect_success and pass
+hello stdout
+hello stderr
+ FAIL print something in test_expect_success and fail
+
+ echo "hello stdout" &&
+ echo "hello stderr" >&2 &&
+ false
+
+hello stdout
+hello stderr
+ PASS print something between test_begin_subtest and test_expect_equal and pass
+hello stdout
+hello stderr
+ FAIL print something test_begin_subtest and test_expect_equal and fail
+ --- test-verbose.4.expected 2010-11-14 21:41:06.650023289 +0000
+ +++ test-verbose.4.output 2010-11-14 21:41:06.650023289 +0000
+ @@ -1 +1 @@
+ -b
+ +a
+
--
1.7.2.3
More information about the notmuch
mailing list