Merge pull request #488 from priyacshah/early_exit_junit_xml_feature

Fix for junit failure FIX #487

Author: fliiiix

.gitignore

@@ -42,3 +42,5 @@ README.rst
 
 # IDE configuration
 .idea/
+.venv/
+.vscode/

radish/extensions/junit_xml_writer.py

@@ -113,31 +113,34 @@ def generate_junit_xml(self, features, marker):
                 if scenario.state is Step.State.FAILED:
                     testsuite_states["failures"] += 1
 
+            if feature.starttime and feature.endtime:
+                feature_duration = feature.duration.total_seconds()
+            else:
+                feature_duration = 0
             testsuite_element = etree.Element(
                 "testsuite",
                 name=feature.sentence,
                 failures=str(testsuite_states["failures"]),
                 errors=str(testsuite_states["errors"]),
                 skipped=str(testsuite_states["skipped"]),
                 tests=str(testsuite_states["tests"]),
-                time="%.3f" % feature.duration.total_seconds(),
+                time=f"{feature_duration:.3f}",
             )
 
             for scenario in (s for s in feature.all_scenarios if not isinstance(s, (ScenarioOutline, ScenarioLoop))):
                 if not scenario.has_to_run(world.config.scenarios):
                     continue
 
-                if scenario.state not in [
-                    Step.State.UNTESTED,
-                    Step.State.PENDING,
-                    Step.State.SKIPPED,
-                ]:
-                    testcase_element = etree.Element(
-                        "testcase",
-                        classname=feature.sentence,
-                        name=scenario.sentence,
-                        time="%.3f" % scenario.duration.total_seconds(),
-                    )
+                if scenario.starttime and scenario.endtime:
+                    testcase_duration = scenario.duration.total_seconds()
+                else:
+                    testcase_duration = 0
+                testcase_element = etree.Element(
+                    "testcase",
+                    classname=feature.sentence,
+                    name=scenario.sentence,
+                    time=f"{testcase_duration:.3f}",
+                )
 
                 if world.config.junit_relaxed:
                     properties_element = etree.Element("properties")
@@ -179,10 +182,5 @@ def generate_junit_xml(self, features, marker):
 
             testsuites_element.append(testsuite_element)
 
-        content = etree.tostring(
-            testsuites_element,
-            pretty_print=True,
-            xml_declaration=True,
-            encoding="utf-8",
-        )
+        content = etree.tostring(testsuites_element, pretty_print=True, xml_declaration=True, encoding="utf-8")
         self._write_xml_to_disk(content)

tests/exploratory/abortedfeature/features/01-SumNumbers.feature

@@ -0,0 +1,18 @@
+Feature: Test summing numbers
+  In order to test the basic
+  features of radish I test
+  to sum numbers.
+
+  Scenario: Sum two numbers
+    Given I have the number 5
+      And I have the number 3
+    When I sum them
+    Then I expect the result to be 8
+
+  Scenario: Sum three numbers
+    Given I have the number 5
+      And I have the number 3
+      And I have the number 2
+    When I sum them
+    Then I expect the result to be 10
+    Then I crash

tests/exploratory/abortedfeature/features/02-NeverRuns.feature

@@ -0,0 +1,17 @@
+Feature: Test summing numbers
+  In order to test the basic
+  features of radish I test
+  to sum numbers.
+
+  Scenario: Sum two numbers
+    Given I have the number 5
+      And I have the number 3
+    When I sum them
+    Then I expect the result to be 8
+
+  Scenario: Sum three numbers
+    Given I have the number 5
+      And I have the number 3
+      And I have the number 2
+    When I sum them
+    Then I expect the result to be 10

tests/exploratory/abortedfeature/radish/steps.py

@@ -0,0 +1,24 @@
+import sys
+
+from radish import then, when
+from radish.stepregistry import step
+
+
+@step("I have the number {number:g}")
+def have_number(step, number):
+    step.context.numbers.append(int(number))
+
+
+@when("I sum them")
+def sum_numbers(step):
+    step.context.result = sum(step.context.numbers)
+
+
+@then("I expect the result to be {result:g}")
+def expect_result(step, result):
+    assert step.context.result == result
+
+
+@then("I crash")
+def crash(step):
+    sys.exit(1)

tests/exploratory/abortedfeature/radish/terrain.py

@@ -0,0 +1,6 @@
+from radish import before
+
+
+@before.each_scenario
+def init_numbers(scenario):
+    scenario.context.numbers = []

tests/unit/extensions/test_junit_xml_writer.py

@@ -7,6 +7,7 @@
 Copyright: MIT, Timo Furrer <[email protected]>
 """
 
+import re
 from datetime import datetime, timezone
 
 import pytest
@@ -16,6 +17,7 @@
 from radish.feature import Feature
 from radish.model import Tag
 from radish.scenario import Scenario
+from radish.stepmodel import Step
 from radish.terrain import world
 
 
@@ -27,7 +29,7 @@ def test_empty_feature_list():
         writer.generate_junit_xml(no_features, "marker-is-ignored")
 
 
-def test_singel_feature_list(mocker):
+def test_single_feature_list(mocker):
     stub = mocker.patch("radish.extensions.junit_xml_writer.JUnitXMLWriter._write_xml_to_disk")
 
     first_feature = Feature(1, "Feature", "I am a feature", "foo.feature", 1, tags=None)
@@ -108,3 +110,33 @@ def test_relaxed_mode_adding_tags_to_junit(mocker):
 
     assert "author" in str(stub.call_args[0])
     assert "batman" in str(stub.call_args[0])
+
+
+def test_early_exit_feature_list(mocker):
+    stub = mocker.patch("radish.extensions.junit_xml_writer.JUnitXMLWriter._write_xml_to_disk")
+
+    first_feature = Feature(1, "Feature", "I am a feature", "foo.feature", 1, tags=None)
+    first_feature.starttime = datetime.now(timezone.utc)
+    first_feature.endtime = datetime.now(timezone.utc)
+    second_feature = Feature(2, "Feature", "Did not run", "foo.feature", 1, tags=None)
+    scenario = Scenario(
+        1, "Scenario", "Did not run", "foo.feature", 1, parent=None, tags=None, preconditions=None, background=None
+    )
+    scenario.steps = [Step(1, "Foo", "foo.feature", 2, None, False)]
+    second_feature.scenarios = [scenario]
+    assert second_feature.state not in [Step.State.PASSED, Step.State.FAILED]
+
+    features = [first_feature, second_feature]
+
+    writer = JUnitXMLWriter()
+    writer.generate_junit_xml(features, "marker-is-ignored")
+
+    result = str(stub.call_args[0])
+    feature_regex = re.compile(r"<testsuite[^>]*name=\"([^\"]+)\"([^>]*)>")
+    matches = feature_regex.findall(result)
+    assert len(matches) == 2
+    f1_match = next(m for m in matches if m[0] == "I am a feature")
+    f2_match = next(m for m in matches if m[0] == "Did not run")
+    assert 'tests="0"' in f1_match[1]  # f1 contains no scenarios
+    assert 'skipped="1"' in f2_match[1]  # f2 contains one untested scenario (it was skipped)
+    assert "<skipped" in result  # there is a skipped testcase element