junitxml: adjust junitxml output file to comply with JUnit xsd

Change XML file structure in the manner that failures in call and errors
in teardown in one test will appear under separate testcase elements in
the XML report.

Author: KKoukiou

CHANGELOG.rst

@@ -87,14 +87,21 @@ Bug Fixes
 3.0.7 (unreleased)
 =======================
 
-*
+* Change junitxml.py to produce reports that comply with Junitxml schema.
+  If the same test fails with failure in call and then errors in teardown
+  we split testcase element into two, one containing the error and the other
+  the failure. (`#2228`_) Thanks to `@kkoukiou`_ for the PR.
 
 *
 
 *
 
 *
 
+.. _@kkoukiou: https://github.com/KKoukiou
+
+.. _#2228:  https://github.com/pytest-dev/pytest/issues/2228
+
 
 3.0.6 (2017-01-22)
 ==================

_pytest/junitxml.py

@@ -273,6 +273,9 @@ def __init__(self, logfile, prefix):
         self.node_reporters = {}  # nodeid -> _NodeReporter
         self.node_reporters_ordered = []
         self.global_properties = []
+        # List of reports that failed on call but teardown is pending.
+        self.open_reports = []
+        self.cnt_double_fail_tests = 0
 
     def finalize(self, report):
         nodeid = getattr(report, 'nodeid', report)
@@ -332,14 +335,33 @@ def pytest_runtest_logreport(self, report):
             -> teardown node2
             -> teardown node1
         """
+        close_report = None
         if report.passed:
             if report.when == "call":  # ignore setup/teardown
                 reporter = self._opentestcase(report)
                 reporter.append_pass(report)
         elif report.failed:
+            if report.when == "teardown":
+                # The following vars are needed when xdist plugin is used
+                report_wid = getattr(report, "worker_id", None)
+                report_ii = getattr(report, "item_index", None)
+                close_report = next(
+                    (rep for rep in self.open_reports
+                     if (rep.nodeid == report.nodeid and
+                         getattr(rep, "item_index", None) == report_ii and
+                         getattr(rep, "worker_id", None) == report_wid
+                         )
+                     ), None)
+                if close_report:
+                    # We need to open new testcase in case we have failure in
+                    # call and error in teardown in order to follow junit
+                    # schema
+                    self.finalize(close_report)
+                    self.cnt_double_fail_tests += 1
             reporter = self._opentestcase(report)
             if report.when == "call":
                 reporter.append_failure(report)
+                self.open_reports.append(report)
             else:
                 reporter.append_error(report)
         elif report.skipped:
@@ -348,6 +370,17 @@ def pytest_runtest_logreport(self, report):
         self.update_testcase_duration(report)
         if report.when == "teardown":
             self.finalize(report)
+            report_wid = getattr(report, "worker_id", None)
+            report_ii = getattr(report, "item_index", None)
+            close_report = next(
+                (rep for rep in self.open_reports
+                 if (rep.nodeid == report.nodeid and
+                      getattr(rep, "item_index", None) == report_ii and
+                      getattr(rep, "worker_id", None) == report_wid
+                     )
+                 ), None)
+            if close_report:
+                self.open_reports.remove(close_report)
 
     def update_testcase_duration(self, report):
         """accumulates total duration for nodeid from given report and updates
@@ -380,8 +413,9 @@ def pytest_sessionfinish(self):
         suite_stop_time = time.time()
         suite_time_delta = suite_stop_time - self.suite_start_time
 
-        numtests = self.stats['passed'] + self.stats['failure'] + self.stats['skipped'] + self.stats['error']
-
+        numtests = (self.stats['passed'] + self.stats['failure'] +
+                    self.stats['skipped'] + self.stats['error'] -
+                    self.cnt_double_fail_tests)
         logfile.write('<?xml version="1.0" encoding="utf-8"?>')
 
         logfile.write(Junit.testsuite(

testing/test_junitxml.py

@@ -189,6 +189,29 @@ def test_function(arg):
         fnode.assert_attr(message="test teardown failure")
         assert "ValueError" in fnode.toxml()
 
+    def test_call_failure_teardown_error(self, testdir):
+        testdir.makepyfile("""
+            import pytest
+
+            @pytest.fixture
+            def arg():
+                yield
+                raise Exception("Teardown Exception")
+            def test_function(arg):
+                raise Exception("Call Exception")
+        """)
+        result, dom = runandparse(testdir)
+        assert result.ret
+        node = dom.find_first_by_tag("testsuite")
+        node.assert_attr(errors=1, failures=1, tests=1)
+        first, second = dom.find_by_tag("testcase")
+        if not first or not second or first == second:
+            assert 0
+        fnode = first.find_first_by_tag("failure")
+        fnode.assert_attr(message="Exception: Call Exception")
+        snode = second.find_first_by_tag("error")
+        snode.assert_attr(message="test teardown failure")
+
     def test_skip_contains_name_reason(self, testdir):
         testdir.makepyfile("""
             import pytest