Allow explicit free variables control at Examples and steps

Author: Kostiantyn Goloveshko

CHANGES.rst

@@ -1,6 +1,11 @@
 Changelog
 =========
 
+Unreleased
+-----------
+- Add options to control free variables in Examples and step definitions
+
+
 5.0.0
 -----
 This release introduces breaking changes, please refer to the :ref:`Migration from 4.x.x`.

README.rst

@@ -696,6 +696,46 @@ With a parametrized.feature file:
 
 
 The significant downside of this approach is inability to see the test table from the feature file.
+It's possible to disallow steps free parameters substitution from fixtures (so test case will fail):
+
+.. code-block:: python
+    @pytest.mark.parametrize(
+        ["start", "eat", "left"],
+        [(12, 5, 7)],
+    )
+    @scenario(
+        "parametrized.feature",
+        "Parametrized given, when, thens",
+        allow_step_free_variables=False,
+    )
+    def test_parametrized(start, eat, left):
+        """We don't need to do anything here, everything will be managed by the scenario decorator."""
+
+Sometimes you want leave a column not used in steps for specific reason in examples section:
+
+.. code-block:: gherkin
+    Feature: Scenario outlines
+        Scenario Outline: Outlined given, when, thens
+            Given there are <start> cucumbers
+            When I eat <eat> cucumbers
+            Then I should have <left> cucumbers
+
+            Examples:
+            | start | eat | left | comment         |
+            |  12   |  5  |  7   | sweet cucumbers!|
+
+
+.. code-block:: python
+    from pytest_bdd import given, when, then, scenario
+
+
+    @scenario(
+        "outline.feature",
+        "Outlined given, when, thens",
+        allow_example_free_variables=True,
+    )
+    def test_outlined():
+        pass
 
 
 Organizing your scenarios

pytest_bdd/parser.py

@@ -232,6 +232,9 @@ def __init__(self, feature: Feature, name: str, line_number: int, tags=None):
         self.line_number = line_number
         self.tags = tags or set()
 
+        self.allow_example_free_variables = None
+        self.allow_step_free_variables = None
+
     def add_step(self, step):
         """Add step to the scenario.
 
@@ -258,18 +261,57 @@ def render(self, context: typing.Mapping[str, typing.Any]) -> "Scenario":
         ]
         return Scenario(feature=self.feature, name=self.name, line_number=self.line_number, steps=steps, tags=self.tags)
 
+    @property
+    def params(self):
+        return frozenset(sum((list(step.params) for step in self.steps), []))
+
+    def get_example_params(self):
+        return set(self.examples.example_params + self.feature.examples.example_params)
+
     def validate(self):
         """Validate the scenario.
 
         :raises ScenarioValidationError: when scenario is not valid
         """
-        params = frozenset(sum((list(step.params) for step in self.steps), []))
-        example_params = set(self.examples.example_params + self.feature.examples.example_params)
-        if params and example_params and params.issubset(example_params):
+        if self.params or self.get_example_params():
+            self._validate_example_free_variables()
+            self._validate_step_free_variables()
+
+    def _validate_example_free_variables(self):
+        params = self.params
+        example_params = self.get_example_params()
+        if self.allow_example_free_variables or example_params.issubset(params):
+            return
+        else:
+            raise exceptions.ScenarioExamplesNotValidError(
+                (
+                    """Scenario "{}" in the feature "{}" does not have valid examples. """
+                    """Set of example parameters {} should be a subset of step """
+                    """parameters {} if examples free variables are not allowed"""
+                ).format(
+                    self.name,
+                    self.feature.filename,
+                    sorted(example_params),
+                    sorted(params),
+                )
+            )
+
+    def _validate_step_free_variables(self):
+        params = self.params
+        example_params = self.get_example_params()
+        if self.allow_step_free_variables or params.issubset(example_params):
+            return
+        else:
             raise exceptions.ScenarioExamplesNotValidError(
-                """Scenario "{}" in the feature "{}" does not have valid examples. """
-                """Set of step parameters {} should be a subset of example values {}.""".format(
-                    self.name, self.feature.filename, sorted(params), sorted(example_params)
+                (
+                    """Scenario "{}" in the feature "{}" does not have valid examples. """
+                    """Set of step parameters {} should be a subset of example """
+                    """parameters {} if steps free variables are not allowed"""
+                ).format(
+                    self.name,
+                    self.feature.filename,
+                    sorted(params),
+                    sorted(example_params),
                 )
             )
 

pytest_bdd/scenario.py

@@ -215,11 +215,21 @@ def collect_example_parametrizations(
     return [pytest.param(context, id="-".join(context.values())) for context in contexts]
 
 
-def scenario(feature_name: str, scenario_name: str, encoding: str = "utf-8", features_base_dir=None):
+def scenario(
+    feature_name: str,
+    scenario_name: str,
+    *,
+    allow_example_free_variables=False,
+    allow_step_free_variables=True,
+    encoding: str = "utf-8",
+    features_base_dir=None,
+):
     """Scenario decorator.
 
     :param str feature_name: Feature file name. Absolute or relative to the configured feature base path.
     :param str scenario_name: Scenario name.
+    :param allow_example_free_variables: Examples could contain free(unused) variables
+    :param allow_step_free_variables: Steps could contain free(unused) variables which could be taken from fixtures
     :param str encoding: Feature file encoding.
     """
 
@@ -234,6 +244,9 @@ def scenario(feature_name: str, scenario_name: str, encoding: str = "utf-8", fea
     # Get the scenario
     try:
         scenario = feature.scenarios[scenario_name]
+        scenario.allow_example_free_variables = allow_example_free_variables
+        scenario.allow_step_free_variables = allow_step_free_variables
+
     except KeyError:
         feature_name = feature.name or "[Empty]"
         raise exceptions.ScenarioNotFound(
@@ -294,10 +307,12 @@ def get_name():
         suffix = f"_{index}"
 
 
-def scenarios(*feature_paths, **kwargs):
+def scenarios(*feature_paths, allow_example_free_variables=False, allow_step_free_variables=True, **kwargs):
     """Parse features from the paths and put all found scenarios in the caller module.
 
     :param *feature_paths: feature file paths to use for scenarios
+    :param allow_example_free_variables: Examples could contain free(unused) variables
+    :param allow_step_free_variables: Steps could contain free(unused) variables which could be taken from fixtures
     """
     caller_locals = get_caller_module_locals()
     caller_path = get_caller_module_path()
@@ -324,7 +339,13 @@ def scenarios(*feature_paths, **kwargs):
             # skip already bound scenarios
             if (scenario_object.feature.filename, scenario_name) not in module_scenarios:
 
-                @scenario(feature.filename, scenario_name, **kwargs)
+                @scenario(
+                    feature.filename,
+                    scenario_name,
+                    allow_example_free_variables=allow_example_free_variables,
+                    allow_step_free_variables=allow_step_free_variables,
+                    **kwargs,
+                )
                 def _scenario():
                     pass  # pragma: no cover
 

tests/feature/test_outline.py

@@ -81,6 +81,54 @@ def test_outline(request):
     # fmt: on
 
 
+def test_disallow_free_example_params(testdir):
+    """Test parametrized scenario when the test function lacks parameters."""
+
+    testdir.makefile(
+        ".feature",
+        outline=textwrap.dedent(
+            """\
+            Feature: Outline
+                Scenario Outline: Outlined with wrong examples
+                    Given there are <start> cucumbers
+                    When I eat <eat> cucumbers
+                    Then I should have <left> cucumbers
+
+                    Examples:
+                    | start | eat | left | unknown_param |
+                    |  12   |  5  |  7   | value         |
+
+            """
+        ),
+    )
+    testdir.makeconftest(textwrap.dedent(STEPS))
+
+    testdir.makepyfile(
+        textwrap.dedent(
+            """\
+        from pytest_bdd import scenario
+
+        @scenario(
+            "outline.feature", 
+            "Outlined with wrong examples",
+            allow_example_free_variables=False
+        )
+        def test_outline(request):
+            pass
+        """
+        )
+    )
+    result = testdir.runpytest()
+    assert_outcomes(result, errors=1)
+    result.stdout.fnmatch_lines(
+        '*ScenarioExamplesNotValidError: Scenario "Outlined with wrong examples"*does not have valid examples*'
+    )
+    result.stdout.fnmatch_lines(
+        "*Set of example parameters [[]'eat', 'left', 'start', 'unknown_param'[]] should be "
+        "a subset of step parameters [[]'eat', 'left', 'start'[]]*"
+    )
+
+
 def test_outline_has_subset_of_parameters(testdir):
     """Test parametrized scenario when the test function has a subset of the parameters of the examples."""
 
@@ -108,8 +156,11 @@ def test_outline_has_subset_of_parameters(testdir):
             """\
         from pytest_bdd import scenario
 
-        @scenario("outline.feature", "Outlined with subset of examples",
-            example_converters=dict(start=int, eat=float, left=str))
+        @scenario(
+            "outline.feature", "
+            Outlined with subset of examples", 
+            allow_example_free_variables=True
+        )
         def test_outline(request):
             pass
         """
@@ -120,7 +171,8 @@ def test_outline(request):
 
 
 def test_wrongly_outlined_parameters_not_a_subset_of_examples(testdir):
-    """Test parametrized scenario when the test function has a parameter set which is not a subset of those in the examples table."""
+    """Test parametrized scenario when the test function has a parameter set
+    which is not a subset of those in the examples table."""
 
     testdir.makefile(
         ".feature",
@@ -145,8 +197,9 @@ def test_wrongly_outlined_parameters_not_a_subset_of_examples(testdir):
         textwrap.dedent(
             """\
         from pytest_bdd import scenario, then
+        import pytest_bdd.parsers as parsers
 
-        @scenario("outline.feature", "Outlined with wrong examples")
+        @scenario("outline.feature", "Outlined with wrong examples", allow_step_free_variables=False)
         def test_outline(request):
             pass
             
@@ -161,7 +214,7 @@ def stepdef(left, right):
     result.stdout.fnmatch_lines(
         '*ScenarioExamplesNotValidError: Scenario "Outlined with wrong examples"*does not have valid examples*',
     )
-    result.stdout.fnmatch_lines("*should be a subset of example values [[]'eat', 'left', 'start'[]].*")
+    result.stdout.fnmatch_lines("*should be a subset of example parameters [[]'eat', 'left', 'start'[]]*")
 
 
 def test_wrong_vertical_examples_scenario(testdir):

tests/feature/test_parametrized.py

@@ -77,3 +77,74 @@ def should_have_left_cucumbers(start_cucumbers, start, eat, left):
         2, 1, 1,
     ]
     # fmt: on
+
+
+def test_outlining_using_fixtures(testdir):
+    """Test parametrized scenario."""
+    testdir.makefile(
+        ".feature",
+        parametrized=textwrap.dedent(
+            """\
+            Feature: Parametrized scenario
+                Scenario: Parametrized given, when, thens
+                    Given there are <start> cucumbers
+                    When I eat <eat> cucumbers
+                    Then I should have <left> cucumbers
+            """
+        ),
+    )
+
+    testdir.makepyfile(
+        textwrap.dedent(
+            """\
+        import pytest
+        from pytest_bdd import given, when, then, scenario
+
+        @pytest.fixture
+        def start():
+            return 12
+            
+        @pytest.fixture
+        def eat():
+            return 5
+            
+        @pytest.fixture
+        def left():
+            return 7
+
+
+        @pytest.fixture(params=[1, 2])
+        def foo_bar(request):
+            return "bar" * request.param
+
+
+        @scenario("parametrized.feature", "Parametrized given, when, thens")
+        def test_parametrized(request, start, eat, left):
+            pass
+
+
+        @scenario("parametrized.feature", "Parametrized given, when, thens")
+        def test_parametrized_with_other_fixtures(request, start, eat, left, foo_bar):
+            pass
+
+        @given("there are <start> cucumbers", target_fixture="start_cucumbers")
+        def start_cucumbers(start):
+            return dict(start=start)
+
+
+        @when("I eat <eat> cucumbers")
+        def eat_cucumbers(start_cucumbers, start, eat):
+            start_cucumbers["eat"] = eat
+
+
+        @then("I should have <left> cucumbers")
+        def should_have_left_cucumbers(start_cucumbers, start, eat, left):
+            assert start - eat == left
+            assert start_cucumbers["start"] == start
+            assert start_cucumbers["eat"] == eat
+
+        """
+        )
+    )
+    result = testdir.runpytest()
+    result.assert_outcomes(passed=3)