[PATCH v2 2/4] test: dynamically generate directive tests

[Jani Nikula]

Similar to the parser test transition to dynamically generate test
methods, but with more test separation added. After this, the Sphinx
build is done independently for each test case, and separately for the
actual directive output and the expected output. This removes any
potential C domain interactions between test cases and expected/actual
inputs.

The change does mean the Sphinx build is run roughly 50x times per full
test run, at the current number of test cases. This is somewhat offset
by the ability to run individual directive test cases:

$ test/test_cautodoc.py DirectiveTest.test_example_10_macro
---
 test/{sphinx => }/conf.py |  0
 test/sphinx/index.rst     | 20 -------------
 test/test_cautodoc.py     | 61 ++++++++++++++-------------------------
 3 files changed, 21 insertions(+), 60 deletions(-)
 rename test/{sphinx => }/conf.py (100%)
 delete mode 100644 test/sphinx/index.rst

diff --git a/test/sphinx/conf.py b/test/conf.py
similarity index 100%
rename from test/sphinx/conf.py
rename to test/conf.py
diff --git a/test/sphinx/index.rst b/test/sphinx/index.rst
deleted file mode 100644
index 20404ce8ee5e..000000000000
--- a/test/sphinx/index.rst
+++ /dev/null
@@ -1,20 +0,0 @@
-.. Hawkmoth Test documentation master file, created by
-   sphinx-quickstart on Wed Dec 12 13:11:26 2018.
-   You can adapt this file completely to your liking, but it should at least
-   contain the root `toctree` directive.
-
-Welcome to Hawkmoth Test's documentation!
-=========================================
-
-.. toctree::
-   :maxdepth: 2
-   :caption: Contents:
-
-
-
-Indices and tables
-==================
-
-* :ref:`genindex`
-* :ref:`modindex`
-* :ref:`search`
diff --git a/test/test_cautodoc.py b/test/test_cautodoc.py
index 3fb98af8629d..848f2105a1da 100755
--- a/test/test_cautodoc.py
+++ b/test/test_cautodoc.py
@@ -9,54 +9,35 @@ import unittest
 import testenv
 from sphinx_testing import with_app
 
-class DirectiveTest(unittest.TestCase):
-
-    def _setup_src(self, srcdir, testcase_in):
-        testcase_out = testenv.modify_filename(testcase_in, dir=srcdir)
-
-        # use the pre-generated rst as comparison data
-        shutil.copyfile(testenv.modify_filename(testcase_in, ext='stdout'),
-                        testenv.modify_filename(testcase_out, ext='expected.rst'))
-
-        # set up an rst file to run the extension
-        shutil.copyfile(testcase_in, testcase_out)
-        options = testenv.get_testcase_options(testcase_in)
+ at with_app(confdir=testenv.testdir, create_new_srcdir=True, buildername='text')
+def _get_output(input_filename, app, status, warning, **options):
+    shutil.copyfile(input_filename,
+                    testenv.modify_filename(input_filename, dir=app.srcdir))
 
-        with open(testenv.modify_filename(testcase_out, ext='output.rst'), 'w') as file:
-            fmt = '.. c:autodoc:: {source}\n'
-            file.write(fmt.format(source=os.path.basename(testcase_out)))
-            for key in options.keys():
-                fmt = '   :{key}: {value}\n'
-                file.write(fmt.format(key=key, value=options[key]))
+    with open(os.path.join(app.srcdir, 'index.rst'), 'w') as file:
+        fmt = '.. c:autodoc:: {source}\n'
+        file.write(fmt.format(source=os.path.basename(input_filename)))
+        for key in options.keys():
+            fmt = '   :{key}: {value}\n'
+            file.write(fmt.format(key=key, value=options[key]))
 
-    def _check_out(self, outdir, testcase_in):
-        testcase_out = testenv.modify_filename(testcase_in, dir=outdir)
+    app.build()
 
-        # compare output from the pre-generated rst against the output generated
-        # by the extension
+    return testenv.read_file(os.path.join(app.outdir, 'index.txt'))
 
-        output = testenv.read_file(testenv.modify_filename(testcase_out,
-                                                           ext='output.txt'))
+ at with_app(confdir=testenv.testdir, create_new_srcdir=True, buildername='text')
+def _get_expected(input_filename, app, status, warning, **options):
+    shutil.copyfile(testenv.modify_filename(input_filename, ext='stdout'),
+                    os.path.join(app.srcdir, 'index.rst'))
 
-        expected = testenv.read_file(testenv.modify_filename(testcase_out,
-                                                             ext='expected.txt'))
+    app.build()
 
-        self.assertEqual(expected, output)
+    return testenv.read_file(os.path.join(app.outdir, 'index.txt'))
 
-    # Use copy_srcdir_to_tmpdir=False and outdir='some-dir' for debugging
-    @with_app(srcdir=os.path.join(testenv.testdir, 'sphinx'),
-              buildername='text', copy_srcdir_to_tmpdir=True)
-    def test_directive(self, app, status, warning):
-        testcases = list(testenv.get_testcases(testenv.testdir))
-
-        for f in testcases:
-            self._setup_src(app.srcdir, f)
-
-        app.build()
+class DirectiveTest(unittest.TestCase):
+    pass
 
-        for f in testcases:
-            with self.subTest(source=os.path.basename(f)):
-                self._check_out(app.outdir, os.path.basename(f))
+testenv.assign_test_methods(DirectiveTest, _get_output, _get_expected)
 
 if __name__ == '__main__':
     unittest.main()
-- 
2.20.1