[PATCH 9/9] test: require test_begin_subtest before test_expect_code
Jani Nikula
jani at nikula.org
Sun Feb 26 05:43:01 PST 2017
Unify the subtests by requiring test_begin_subtest before
test_expect_code. (Similar change for test_expect_success has already
been done.)
This increases clarity in the test scripts by having a separate line
for the start of the subtest with the heading, and makes it possible
to simplify the test infrastructure by making all subtests similar.
---
test/README | 5 +++++
test/T000-basic.sh | 11 ++++-------
test/T020-compact.sh | 3 ++-
test/T050-new.sh | 4 ++--
test/T070-insert.sh | 34 ++++++++++++++++++++--------------
test/T150-tagging.sh | 13 +++++++++----
test/T160-json.sh | 8 ++++----
test/T400-hooks.sh | 14 ++++++++------
test/T570-revision-tracking.sh | 16 ++++++++--------
test/T600-named-queries.sh | 4 ++--
test/test-lib.sh | 18 +++++++++++-------
11 files changed, 75 insertions(+), 55 deletions(-)
diff --git a/test/README b/test/README
index 7acdb4b81f4d..dcd05237a062 100644
--- a/test/README
+++ b/test/README
@@ -200,6 +200,11 @@ library for your script to use.
<script>. If it yields success, test is considered
successful.
+ test_expect_code <code> <script>
+
+ This takes two strings as parameter, and evaluates the <script>.
+ If it yields <code> exit status, test is considered successful.
+
test_subtest_known_broken
Mark the current test as broken. Such tests are expected to fail.
diff --git a/test/T000-basic.sh b/test/T000-basic.sh
index 003ab95f5681..36a7ca4c5bdf 100755
--- a/test/T000-basic.sh
+++ b/test/T000-basic.sh
@@ -30,11 +30,9 @@ test_begin_subtest 'tests clean up after themselves'
clean=no
test_expect_success 'test_when_finished clean=yes'
+test_begin_subtest 'tests clean up even after a failure'
cleaner=no
-test_expect_code 1 'tests clean up even after a failure' '
- test_when_finished cleaner=yes &&
- (exit 1)
-'
+test_expect_code 1 'test_when_finished cleaner=yes && (exit 1)'
if test $clean$cleaner != yesyes
then
@@ -42,9 +40,8 @@ then
exit 1
fi
-test_expect_code 2 'failure to clean up causes the test to fail' '
- test_when_finished "(exit 2)"
-'
+test_begin_subtest 'failure to clean up causes the test to fail'
+test_expect_code 2 'test_when_finished "(exit 2)"'
EXPECTED=$TEST_DIRECTORY/test.expected-output
suppress_diff_date() {
diff --git a/test/T020-compact.sh b/test/T020-compact.sh
index 1fecd071e77e..a3d7380e81c8 100755
--- a/test/T020-compact.sh
+++ b/test/T020-compact.sh
@@ -16,7 +16,8 @@ if [ $NOTMUCH_HAVE_XAPIAN_COMPACT -eq 0 ]; then
test_expect_equal "$output" "notmuch was compiled against a xapian version lacking compaction support.
Compaction failed: Unsupported operation"
- test_expect_code 1 "Compact unsupported: status code" "notmuch compact"
+ test_begin_subtest "Compact unsupported: status code"
+ test_expect_code 1 "notmuch compact"
test_done
fi
diff --git a/test/T050-new.sh b/test/T050-new.sh
index 9115de820770..ffa303efacec 100755
--- a/test/T050-new.sh
+++ b/test/T050-new.sh
@@ -286,8 +286,8 @@ notmuch config set new.tags "-foo;bar"
output=$(NOTMUCH_NEW --debug 2>&1)
test_expect_equal "$output" "Error: tag '-foo' in new.tags: tag starting with '-' forbidden"
-test_expect_code 1 "Invalid tags set exit code" \
- "NOTMUCH_NEW --debug 2>&1"
+test_begin_subtest "Invalid tags set exit code"
+test_expect_code 1 "NOTMUCH_NEW --debug 2>&1"
notmuch config set new.tags $OLDCONFIG
diff --git a/test/T070-insert.sh b/test/T070-insert.sh
index 9120debabf8c..48f212ee0928 100755
--- a/test/T070-insert.sh
+++ b/test/T070-insert.sh
@@ -20,13 +20,13 @@ gen_insert_msg() {
"[body]=\"insert-message\""
}
-test_expect_code 1 "Insert zero-length file" \
- "notmuch insert < /dev/null"
+test_begin_subtest "Insert zero-length file"
+test_expect_code 1 "notmuch insert < /dev/null"
# This test is a proxy for other errors that may occur while trying to
# add a message to the notmuch database, e.g. database locked.
-test_expect_code 1 "Insert non-message" \
- "echo bad_message | notmuch insert"
+test_begin_subtest "Insert non-message"
+test_expect_code 1 "echo bad_message | notmuch insert"
test_begin_subtest "Database empty so far"
test_expect_equal "0" "`notmuch count --output=messages '*'`"
@@ -138,9 +138,9 @@ notmuch insert --folder=Drafts +draft -unread < "$gen_msg_filename"
output=$(notmuch search --output=messages path:Drafts/cur tag:draft NOT tag:unread)
test_expect_equal "$output" "id:$gen_msg_id"
+test_begin_subtest "Insert message into non-existent folder"
gen_insert_msg
-test_expect_code 1 "Insert message into non-existent folder" \
- "notmuch insert --folder=nonesuch < $gen_msg_filename"
+test_expect_code 1 "notmuch insert --folder=nonesuch < $gen_msg_filename"
test_begin_subtest "Insert message, create folder"
gen_insert_msg
@@ -162,9 +162,9 @@ notmuch insert --folder=F/G/H/I/J --create-folder +folder < "$gen_msg_filename"
output=$(notmuch count path:F/G/H/I/J/new tag:folder)
test_expect_equal "$output" "2"
+test_begin_subtest "Insert message, create invalid subfolder"
gen_insert_msg
-test_expect_code 1 "Insert message, create invalid subfolder" \
- "notmuch insert --folder=../G --create-folder $gen_msg_filename"
+test_expect_code 1 "notmuch insert --folder=../G --create-folder $gen_msg_filename"
OLDCONFIG=$(notmuch config get new.tags)
@@ -180,8 +180,8 @@ gen_insert_msg
output=$(notmuch insert $gen_msg_filename 2>&1)
test_expect_equal "$output" "Error: tag '-foo' in new.tags: tag starting with '-' forbidden"
-test_expect_code 1 "Invalid tags set exit code" \
- "notmuch insert $gen_msg_filename 2>&1"
+test_begin_subtest "Invalid tags set exit code"
+test_expect_code 1 "notmuch insert $gen_msg_filename 2>&1"
notmuch config set new.tags $OLDCONFIG
@@ -205,22 +205,28 @@ done
gen_insert_msg
for code in FILE_NOT_EMAIL READ_ONLY_DATABASE UPGRADE_REQUIRED PATH_ERROR; do
- test_expect_code 1 "EXIT_FAILURE when add_message returns $code" \
+ test_begin_subtest "EXIT_FAILURE when add_message returns $code"
+ test_expect_code 1 \
"${TEST_GDB} --batch-silent --return-child-result \
-ex 'set args insert < $gen_msg_filename' \
-x index-file-$code.gdb notmuch"
- test_expect_code 0 "success exit with --keep when add_message returns $code" \
+
+ test_begin_subtest "success exit with --keep when add_message returns $code"
+ test_expect_code 0 \
"${TEST_GDB} --batch-silent --return-child-result \
-ex 'set args insert --keep < $gen_msg_filename' \
-x index-file-$code.gdb notmuch"
done
for code in OUT_OF_MEMORY XAPIAN_EXCEPTION ; do
- test_expect_code 75 "EX_TEMPFAIL when add_message returns $code" \
+ test_begin_subtest "EX_TEMPFAIL when add_message returns $code"
+ test_expect_code 75 \
"${TEST_GDB} --batch-silent --return-child-result \
-ex 'set args insert < $gen_msg_filename' \
-x index-file-$code.gdb notmuch"
- test_expect_code 0 "success exit with --keep when add_message returns $code" \
+
+ test_begin_subtest "success exit with --keep when add_message returns $code"
+ test_expect_code 0 \
"${TEST_GDB} --batch-silent --return-child-result \
-ex 'set args insert --keep < $gen_msg_filename' \
-x index-file-$code.gdb notmuch"
diff --git a/test/T150-tagging.sh b/test/T150-tagging.sh
index 61d1311608a7..0d0a3b874526 100755
--- a/test/T150-tagging.sh
+++ b/test/T150-tagging.sh
@@ -19,8 +19,11 @@ test_expect_equal "$output" "\
thread:XXX 2001-01-05 [1/1] Notmuch Test Suite; One (inbox tag3 unread)
thread:XXX 2001-01-05 [1/1] Notmuch Test Suite; Two (inbox tag3 unread)"
-test_expect_code 1 "No tag operations" 'notmuch tag One'
-test_expect_code 1 "No query" 'notmuch tag +tag2'
+test_begin_subtest "No tag operations"
+test_expect_code 1 'notmuch tag One'
+
+test_begin_subtest "No query"
+test_expect_code 1 'notmuch tag +tag2'
test_begin_subtest "Redundant tagging"
notmuch tag +tag1 -tag3 One
@@ -282,9 +285,11 @@ notmuch dump --format=batch-tag| \
test_expect_equal_file EXPECTED OUTPUT
-test_expect_code 1 "Empty tag names" 'notmuch tag + One'
+test_begin_subtest "Empty tag names"
+test_expect_code 1 'notmuch tag + One'
-test_expect_code 1 "Tag name beginning with -" 'notmuch tag +- One'
+test_begin_subtest "Tag name beginning with -"
+test_expect_code 1 'notmuch tag +- One'
test_begin_subtest "Xapian exception: read only files"
chmod u-w ${MAIL_DIR}/.notmuch/xapian/*.${db_ending}
diff --git a/test/T160-json.sh b/test/T160-json.sh
index 85b9b41f314c..48d30019e44a 100755
--- a/test/T160-json.sh
+++ b/test/T160-json.sh
@@ -64,11 +64,11 @@ test_expect_equal_json "$output" "[{\"thread\": \"XXX\",
\"tags\": [\"inbox\",
\"unread\"]}]"
-test_expect_code 20 "Format version: too low" \
- "notmuch search --format-version=0 \\*"
+test_begin_subtest "Format version: too low"
+test_expect_code 20 "notmuch search --format-version=0 \\*"
-test_expect_code 21 "Format version: too high" \
- "notmuch search --format-version=999 \\*"
+test_begin_subtest "Format version: too high"
+test_expect_code 21 "notmuch search --format-version=999 \\*"
test_begin_subtest "Show message: multiple filenames"
add_message "[id]=message-id at example.com [filename]=copy1"
diff --git a/test/T400-hooks.sh b/test/T400-hooks.sh
index ae571ba282d9..7917a82f0154 100755
--- a/test/T400-hooks.sh
+++ b/test/T400-hooks.sh
@@ -70,7 +70,8 @@ output=`notmuch new 2>&1`
test_expect_equal "$output" "Error: pre-new hook failed with status 13"
# depends on the previous subtest leaving broken hook behind
-test_expect_code 1 "pre-new non-zero exit status (notmuch status)" "notmuch new"
+test_begin_subtest "pre-new non-zero exit status (notmuch status)"
+test_expect_code 1 "notmuch new"
# depends on the previous subtests leaving 1 new message behind
test_begin_subtest "pre-new non-zero exit status aborts new"
@@ -89,7 +90,8 @@ echo "Error: post-new hook failed with status 13" >> expected
test_expect_equal_file expected output
# depends on the previous subtest leaving broken hook behind
-test_expect_code 1 "post-new non-zero exit status (notmuch status)" "notmuch new"
+test_begin_subtest "post-new non-zero exit status (notmuch status)"
+test_expect_code 1 "notmuch new"
test_begin_subtest "post-insert hook does not affect insert status"
rm_hooks
@@ -97,7 +99,7 @@ generate_message
create_failing_hook "post-insert"
test_expect_success "notmuch insert < \"$gen_msg_filename\" > /dev/null"
-# test_begin_subtest "hook without executable permissions"
+test_begin_subtest "hook without executable permissions"
rm_hooks
mkdir -p ${HOOK_DIR}
cat <<EOF >"${HOOK_DIR}/pre-new"
@@ -105,15 +107,15 @@ cat <<EOF >"${HOOK_DIR}/pre-new"
echo foo
EOF
output=`notmuch new 2>&1`
-test_expect_code 1 "hook without executable permissions" "notmuch new"
+test_expect_code 1 "notmuch new"
-# test_begin_subtest "hook execution failure"
+test_begin_subtest "hook execution failure"
rm_hooks
mkdir -p ${HOOK_DIR}
cat <<EOF >"${HOOK_DIR}/pre-new"
no hashbang, execl fails
EOF
chmod +x "${HOOK_DIR}/pre-new"
-test_expect_code 1 "hook execution failure" "notmuch new"
+test_expect_code 1 "notmuch new"
test_done
diff --git a/test/T570-revision-tracking.sh b/test/T570-revision-tracking.sh
index a63e691cd3d4..76ad227944a0 100755
--- a/test/T570-revision-tracking.sh
+++ b/test/T570-revision-tracking.sh
@@ -55,23 +55,23 @@ test_expect_success "notmuch search --uuid=$(cat UUID) '*'"
test_begin_subtest "uuid works as global option"
test_expect_success "notmuch --uuid=$(cat UUID) search '*'"
-test_expect_code 1 'uuid works as global option II' \
- "notmuch --uuid=this-is-no-uuid search '*'"
+test_begin_subtest "uuid works as global option II"
+test_expect_code 1 "notmuch --uuid=this-is-no-uuid search '*'"
-test_expect_code 1 'search fails with incorrect uuid' \
- "notmuch search --uuid=this-is-no-uuid '*'"
+test_begin_subtest "search fails with incorrect uuid"
+test_expect_code 1 "notmuch search --uuid=this-is-no-uuid '*'"
test_begin_subtest "show succeeds with correct uuid"
test_expect_success "notmuch show --uuid=$(cat UUID) '*'"
-test_expect_code 1 'show fails with incorrect uuid' \
- "notmuch show --uuid=this-is-no-uuid '*'"
+test_begin_subtest "show fails with incorrect uuid"
+test_expect_code 1 "notmuch show --uuid=this-is-no-uuid '*'"
test_begin_subtest "tag succeeds with correct uuid"
test_expect_success "notmuch tag --uuid=$(cat UUID) +test '*'"
-test_expect_code 1 'tag fails with incorrect uuid' \
- "notmuch tag --uuid=this-is-no-uuid '*' +test2"
+test_begin_subtest "tag fails with incorrect uuid"
+test_expect_code 1 "notmuch tag --uuid=this-is-no-uuid '*' +test2"
test_begin_subtest 'lastmod:0.. matches everything'
total=$(notmuch count '*')
diff --git a/test/T600-named-queries.sh b/test/T600-named-queries.sh
index 450da9b0c25a..495b7699a0a7 100755
--- a/test/T600-named-queries.sh
+++ b/test/T600-named-queries.sh
@@ -4,8 +4,8 @@ test_description='named queries'
QUERYSTR="date:2009-11-18..2009-11-18 and tag:unread"
-test_expect_code 1 "error adding named query before initializing DB" \
- "notmuch config set query.test \"$QUERYSTR\""
+test_begin_subtest "error adding named query before initializing DB"
+test_expect_code 1 "notmuch config set query.test \"$QUERYSTR\""
add_email_corpus
diff --git a/test/test-lib.sh b/test/test-lib.sh
index a3a4cdcd8a8b..865afd5ecc33 100644
--- a/test/test-lib.sh
+++ b/test/test-lib.sh
@@ -941,13 +941,17 @@ test_expect_success () {
}
test_expect_code () {
- test "$#" = 3 ||
- error "bug in the test script: not 3 parameters to test_expect_code"
- test_subtest_name="$2"
- test_reset_state_
- if ! test_skip "$@"
+ exec 1>&6 2>&7 # Restore stdout and stderr
+ if [ -z "$inside_subtest" ]; then
+ error "bug in the test script: test_expect_code without test_begin_subtest"
+ fi
+ inside_subtest=
+ test "$#" = 2 ||
+ error "bug in the test script: not 2 parameters to test_expect_code"
+
+ if ! test_skip "$test_subtest_name"
then
- test_run_ "$3"
+ test_run_ "$2"
run_ret="$?"
# test_run_ may update missing external prerequisites,
test_check_missing_external_prereqs_ "$@" ||
@@ -955,7 +959,7 @@ test_expect_code () {
then
test_ok_
else
- test_failure_ "exit code $eval_ret, expected $1" "$3"
+ test_failure_ "exit code $eval_ret, expected $1" "$2"
fi
fi
}
--
2.11.0
More information about the notmuch
mailing list