diff --git a/pandas/core/computation/expressions.py b/pandas/core/computation/expressions.py
index 957a493925405..a8852ae06f578 100644
--- a/pandas/core/computation/expressions.py
+++ b/pandas/core/computation/expressions.py
@@ -114,6 +114,11 @@ def _evaluate_numexpr(op, op_str, a, b):
             # numexpr raises eg for array ** array with integers
             # (https://github.com/pydata/numexpr/issues/379)
             pass
+        except NotImplementedError:
+            if _bool_arith_fallback(op_str, a, b):
+                pass
+            else:
+                raise
 
         if is_reversed:
             # reverse order to original for fallback
@@ -197,26 +202,24 @@ def _has_bool_dtype(x):
         return isinstance(x, (bool, np.bool_))
 
 
-def _bool_arith_check(
-    op_str, a, b, not_allowed=frozenset(("/", "//", "**")), unsupported=None
-):
-    if unsupported is None:
-        unsupported = {"+": "|", "*": "&", "-": "^"}
+_BOOL_OP_UNSUPPORTED = {"+": "|", "*": "&", "-": "^"}
 
+
+def _bool_arith_fallback(op_str, a, b):
+    """
+    Check if we should fallback to the python `_evaluate_standard` in case
+    of an unsupported operation by numexpr, which is the case for some
+    boolean ops.
+    """
     if _has_bool_dtype(a) and _has_bool_dtype(b):
-        if op_str in unsupported:
+        if op_str in _BOOL_OP_UNSUPPORTED:
             warnings.warn(
                 f"evaluating in Python space because the {repr(op_str)} "
-                "operator is not supported by numexpr for "
-                f"the bool dtype, use {repr(unsupported[op_str])} instead"
+                "operator is not supported by numexpr for the bool dtype, "
+                f"use {repr(_BOOL_OP_UNSUPPORTED[op_str])} instead"
             )
-            return False
-
-        if op_str in not_allowed:
-            raise NotImplementedError(
-                f"operator {repr(op_str)} not implemented for bool dtypes"
-            )
-    return True
+            return True
+    return False
 
 
 def evaluate(op, a, b, use_numexpr: bool = True):
@@ -233,7 +236,6 @@ def evaluate(op, a, b, use_numexpr: bool = True):
     """
     op_str = _op_str_mapping[op]
     if op_str is not None:
-        use_numexpr = use_numexpr and _bool_arith_check(op_str, a, b)
         if use_numexpr:
             # error: "None" not callable
             return _evaluate(op, op_str, a, b)  # type: ignore[misc]
diff --git a/pandas/core/ops/array_ops.py b/pandas/core/ops/array_ops.py
index 67b68ce7365cc..2ff93b203a001 100644
--- a/pandas/core/ops/array_ops.py
+++ b/pandas/core/ops/array_ops.py
@@ -218,6 +218,10 @@ def arithmetic_op(left: ArrayLike, right: Any, op):
         # because numexpr will fail on it, see GH#31457
         res_values = op(left, right)
     else:
+        # TODO we should handle EAs consistently and move this check before the if/else
+        # (https://github.com/pandas-dev/pandas/issues/41165)
+        _bool_arith_check(op, left, right)
+
         res_values = _na_arithmetic_op(left, right, op)
 
     return res_values
@@ -492,3 +496,28 @@ def _maybe_upcast_for_op(obj, shape: Shape):
         return Timedelta(obj)
 
     return obj
+
+
+_BOOL_OP_NOT_ALLOWED = {
+    operator.truediv,
+    roperator.rtruediv,
+    operator.floordiv,
+    roperator.rfloordiv,
+    operator.pow,
+    roperator.rpow,
+}
+
+
+def _bool_arith_check(op, a, b):
+    """
+    In contrast to numpy, pandas raises an error for certain operations
+    with booleans.
+    """
+    if op in _BOOL_OP_NOT_ALLOWED:
+        if is_bool_dtype(a.dtype) and (
+            is_bool_dtype(b) or isinstance(b, (bool, np.bool_))
+        ):
+            op_name = op.__name__.strip("_").lstrip("r")
+            raise NotImplementedError(
+                f"operator '{op_name}' not implemented for bool dtypes"
+            )
diff --git a/pandas/tests/frame/test_arithmetic.py b/pandas/tests/frame/test_arithmetic.py
index 4c31d15541412..90a9bede40a6b 100644
--- a/pandas/tests/frame/test_arithmetic.py
+++ b/pandas/tests/frame/test_arithmetic.py
@@ -941,16 +941,16 @@ def test_binop_other(self, op, value, dtype):
         elif (op, dtype) in skip:
 
             if op in [operator.add, operator.mul]:
-                with tm.assert_produces_warning(UserWarning):
-                    # "evaluating in Python space because ..."
-                    op(s, e.value)
+                # TODO we should assert this or not depending on whether
+                # numexpr is used or not
+                # with tm.assert_produces_warning(UserWarning):
+                #     # "evaluating in Python space because ..."
+                op(s, e.value)
 
             else:
                 msg = "operator '.*' not implemented for .* dtypes"
                 with pytest.raises(NotImplementedError, match=msg):
-                    with tm.assert_produces_warning(UserWarning):
-                        # "evaluating in Python space because ..."
-                        op(s, e.value)
+                    op(s, e.value)
 
         else:
             # FIXME: Since dispatching to Series, this test no longer
diff --git a/pandas/tests/test_expressions.py b/pandas/tests/test_expressions.py
index e94cb23b359d0..6ac85f9d36fdc 100644
--- a/pandas/tests/test_expressions.py
+++ b/pandas/tests/test_expressions.py
@@ -242,7 +242,7 @@ def testit():
     def test_bool_ops_raise_on_arithmetic(self, op_str, opname):
         df = DataFrame({"a": np.random.rand(10) > 0.5, "b": np.random.rand(10) > 0.5})
 
-        msg = f"operator {repr(op_str)} not implemented for bool dtypes"
+        msg = f"operator '{opname}' not implemented for bool dtypes"
         f = getattr(operator, opname)
         err_msg = re.escape(msg)