diff --git a/pandas/core/groupby/generic.py b/pandas/core/groupby/generic.py
index 2ad85903b916b..ea2bd22cccc3d 100644
--- a/pandas/core/groupby/generic.py
+++ b/pandas/core/groupby/generic.py
@@ -21,7 +21,11 @@
 from pandas.errors import AbstractMethodError
 from pandas.util._decorators import Appender, Substitution
 
-from pandas.core.dtypes.cast import maybe_convert_objects, maybe_downcast_to_dtype
+from pandas.core.dtypes.cast import (
+    maybe_convert_objects,
+    maybe_downcast_numeric,
+    maybe_downcast_to_dtype,
+)
 from pandas.core.dtypes.common import (
     ensure_int64,
     ensure_platform_int,
@@ -180,10 +184,8 @@ def _cython_agg_blocks(self, how, alt=None, numeric_only=True, min_count=-1):
                     continue
             finally:
                 if result is not no_result:
-                    dtype = block.values.dtype
-
                     # see if we can cast the block back to the original dtype
-                    result = block._try_coerce_and_cast_result(result, dtype=dtype)
+                    result = maybe_downcast_numeric(result, block.dtype)
                     newb = block.make_block(result)
 
             new_items.append(locs)
diff --git a/pandas/core/groupby/ops.py b/pandas/core/groupby/ops.py
index 676f243c9c8d3..b0c629f017dd3 100644
--- a/pandas/core/groupby/ops.py
+++ b/pandas/core/groupby/ops.py
@@ -591,6 +591,8 @@ def _cython_operation(self, kind, values, how, axis, min_count=-1, **kwargs):
 
         if is_datetime64tz_dtype(orig_values.dtype):
             result = type(orig_values)(result.astype(np.int64), dtype=orig_values.dtype)
+        elif is_datetimelike and kind == "aggregate":
+            result = result.astype(orig_values.dtype)
 
         return result, names
 
diff --git a/pandas/core/internals/blocks.py b/pandas/core/internals/blocks.py
index 9f3aa699cfaf4..8c3cf7cc51495 100644
--- a/pandas/core/internals/blocks.py
+++ b/pandas/core/internals/blocks.py
@@ -18,6 +18,7 @@
     find_common_type,
     infer_dtype_from,
     infer_dtype_from_scalar,
+    maybe_downcast_numeric,
     maybe_downcast_to_dtype,
     maybe_infer_dtype_type,
     maybe_promote,
@@ -55,7 +56,6 @@
     ABCDataFrame,
     ABCDatetimeIndex,
     ABCExtensionArray,
-    ABCIndexClass,
     ABCPandasArray,
     ABCSeries,
 )
@@ -685,28 +685,6 @@ def _can_hold_element(self, element):
             return issubclass(tipo.type, dtype)
         return isinstance(element, dtype)
 
-    def _try_cast_result(self, result, dtype=None):
-        """ try to cast the result to our original type, we may have
-        roundtripped thru object in the mean-time
-        """
-        if dtype is None:
-            dtype = self.dtype
-
-        if self.is_integer or self.is_bool or self.is_datetime:
-            pass
-        elif self.is_float and result.dtype == self.dtype:
-            # protect against a bool/object showing up here
-            if isinstance(dtype, str) and dtype == "infer":
-                return result
-
-            # This is only reached via Block.setitem, where dtype is always
-            #  either "infer", self.dtype, or values.dtype.
-            assert dtype == self.dtype, (dtype, self.dtype)
-            return result
-
-        # may need to change the dtype here
-        return maybe_downcast_to_dtype(result, dtype)
-
     def _try_coerce_args(self, other):
         """ provide coercion to our input arguments """
 
@@ -729,10 +707,6 @@ def _try_coerce_args(self, other):
 
         return other
 
-    def _try_coerce_and_cast_result(self, result, dtype=None):
-        result = self._try_cast_result(result, dtype=dtype)
-        return result
-
     def to_native_types(self, slicer=None, na_rep="nan", quoting=None, **kwargs):
         """ convert to our native types format, slicing if desired """
 
@@ -925,8 +899,6 @@ def setitem(self, indexer, value):
         else:
             values[indexer] = value
 
-        # coerce and try to infer the dtypes of the result
-        values = self._try_coerce_and_cast_result(values, dtype)
         if transpose:
             values = values.T
         block = self.make_block(values)
@@ -1444,10 +1416,6 @@ def func(cond, values, other):
             if transpose:
                 result = result.T
 
-            # try to cast if requested
-            if try_cast:
-                result = self._try_cast_result(result)
-
             return [self.make_block(result)]
 
         # might need to separate out blocks
@@ -1459,7 +1427,7 @@ def func(cond, values, other):
         for m in [mask, ~mask]:
             if m.any():
                 taken = result.take(m.nonzero()[0], axis=axis)
-                r = self._try_cast_result(taken)
+                r = maybe_downcast_numeric(taken, self.dtype)
                 nb = self.make_block(r.T, placement=self.mgr_locs[m])
                 result_blocks.append(nb)
 
@@ -1692,9 +1660,6 @@ def putmask(self, mask, new, align=True, inplace=False, axis=0, transpose=False)
         new_values[mask] = new
         return [self.make_block(values=new_values)]
 
-    def _try_cast_result(self, result, dtype=None):
-        return result
-
     def _get_unstack_items(self, unstacker, new_columns):
         """
         Get the placement, values, and mask for a Block unstack.
@@ -1746,7 +1711,8 @@ def __init__(self, values, placement, ndim=None):
         super().__init__(values, placement, ndim)
 
     def _maybe_coerce_values(self, values):
-        """Unbox to an extension array.
+        """
+        Unbox to an extension array.
 
         This will unbox an ExtensionArray stored in an Index or Series.
         ExtensionArrays pass through. No dtype coercion is done.
@@ -1759,9 +1725,7 @@ def _maybe_coerce_values(self, values):
         -------
         ExtensionArray
         """
-        if isinstance(values, (ABCIndexClass, ABCSeries)):
-            values = values._values
-        return values
+        return extract_array(values)
 
     @property
     def _holder(self):