diff --git a/doc/source/whatsnew/v2.0.0.rst b/doc/source/whatsnew/v2.0.0.rst
index 29f360e050548..c991b3b6a6004 100644
--- a/doc/source/whatsnew/v2.0.0.rst
+++ b/doc/source/whatsnew/v2.0.0.rst
@@ -1250,6 +1250,8 @@ Indexing
 - Bug in :meth:`Series.loc` raising error for out of bounds end of slice indexer (:issue:`50161`)
 - Bug in :meth:`DataFrame.loc` raising ``ValueError`` with ``bool`` indexer and :class:`MultiIndex` (:issue:`47687`)
 - Bug in :meth:`DataFrame.loc` raising ``IndexError`` when setting values for a pyarrow-backed column with a non-scalar indexer (:issue:`50085`)
+- Bug in :meth:`DataFrame.__getitem__`, :meth:`Series.__getitem__`, :meth:`DataFrame.__setitem__` and :meth:`Series.__setitem__`
+  when indexing on indexes with extension float dtypes (:class:`Float64` & :class:`Float64`) or complex dtypes using integers (:issue:`51053`)
 - Bug in :meth:`DataFrame.loc` modifying object when setting incompatible value with an empty indexer (:issue:`45981`)
 - Bug in :meth:`DataFrame.__setitem__` raising ``ValueError`` when right hand side is :class:`DataFrame` with :class:`MultiIndex` columns (:issue:`49121`)
 - Bug in :meth:`DataFrame.reindex` casting dtype to ``object`` when :class:`DataFrame` has single extension array column when re-indexing ``columns`` and ``index`` (:issue:`48190`)
diff --git a/pandas/core/indexes/base.py b/pandas/core/indexes/base.py
index 363bfe76d40fb..9d44c375c0b80 100644
--- a/pandas/core/indexes/base.py
+++ b/pandas/core/indexes/base.py
@@ -5680,9 +5680,12 @@ def _should_fallback_to_positional(self) -> bool:
         """
         Should an integer key be treated as positional?
         """
-        if isinstance(self.dtype, np.dtype) and self.dtype.kind in ["i", "u", "f"]:
-            return False
-        return not self._holds_integer()
+        return self.inferred_type not in {
+            "integer",
+            "mixed-integer",
+            "floating",
+            "complex",
+        }
 
     _index_shared_docs[
         "get_indexer_non_unique"
diff --git a/pandas/tests/frame/indexing/test_indexing.py b/pandas/tests/frame/indexing/test_indexing.py
index 2dfc31ccc1638..54e30a3355943 100644
--- a/pandas/tests/frame/indexing/test_indexing.py
+++ b/pandas/tests/frame/indexing/test_indexing.py
@@ -57,6 +57,15 @@ def test_getitem(self, float_frame):
         with pytest.raises(KeyError, match="random"):
             float_frame["random"]
 
+    def test_getitem_numeric_should_not_fallback_to_positional(self, any_numeric_dtype):
+        # GH51053
+        dtype = any_numeric_dtype
+        idx = Index([1, 0, 1], dtype=dtype)
+        df = DataFrame([[1, 2, 3], [4, 5, 6]], columns=idx)
+        result = df[1]
+        expected = DataFrame([[1, 3], [4, 6]], columns=Index([1, 1], dtype=dtype))
+        tm.assert_frame_equal(result, expected, check_exact=True)
+
     def test_getitem2(self, float_frame):
 
         df = float_frame.copy()
@@ -71,6 +80,15 @@ def test_getitem2(self, float_frame):
         res = df["@awesome_domain"]
         tm.assert_numpy_array_equal(ad, res.values)
 
+    def test_setitem_numeric_should_not_fallback_to_positional(self, any_numeric_dtype):
+        # GH51053
+        dtype = any_numeric_dtype
+        idx = Index([1, 0, 1], dtype=dtype)
+        df = DataFrame([[1, 2, 3], [4, 5, 6]], columns=idx)
+        df[1] = 10
+        expected = DataFrame([[10, 2, 10], [10, 5, 10]], columns=idx)
+        tm.assert_frame_equal(df, expected, check_exact=True)
+
     def test_setitem_list(self, float_frame):
 
         float_frame["E"] = "foo"
diff --git a/pandas/tests/series/indexing/test_indexing.py b/pandas/tests/series/indexing/test_indexing.py
index f214ade0a31aa..a8290f472cd7c 100644
--- a/pandas/tests/series/indexing/test_indexing.py
+++ b/pandas/tests/series/indexing/test_indexing.py
@@ -45,6 +45,26 @@ def test_basic_indexing():
         s[5] = 0
 
 
+def test_getitem_numeric_should_not_fallback_to_positional(any_numeric_dtype):
+    # GH51053
+    dtype = any_numeric_dtype
+    idx = Index([1, 0, 1], dtype=dtype)
+    ser = Series(range(3), index=idx)
+    result = ser[1]
+    expected = Series([0, 2], index=Index([1, 1], dtype=dtype))
+    tm.assert_series_equal(result, expected, check_exact=True)
+
+
+def test_setitem_numeric_should_not_fallback_to_positional(any_numeric_dtype):
+    # GH51053
+    dtype = any_numeric_dtype
+    idx = Index([1, 0, 1], dtype=dtype)
+    ser = Series(range(3), index=idx)
+    ser[1] = 10
+    expected = Series([10, 1, 10], index=idx)
+    tm.assert_series_equal(ser, expected, check_exact=True)
+
+
 def test_basic_getitem_with_labels(datetime_series):
     indices = datetime_series.index[[5, 10, 15]]