BUG: SparseSeries.value_counts ignores fill_value

closes #6749

Author: sinhrks <[email protected]>

Closes #12835 from sinhrks/sparse_valuecounts and squashes the following commits:

2392e7c [sinhrks] Move dtype handling to algorithm
2f46f73 [sinhrks] BUG: SparseSeries.value_counts ignores fill_value

Author: sinhrks

doc/source/whatsnew/v0.18.1.txt

@@ -259,6 +259,7 @@ These changes conform sparse handling to return the correct types and work to ma
 - Bug in ``SparseSeries`` and ``SparseArray`` may have different ``dtype`` from its dense values (:issue:`12908`)
 - Bug in ``SparseSeries.reindex`` incorrectly handle ``fill_value`` (:issue:`12797`)
 - Bug in ``SparseArray.to_frame()`` results in ``DataFrame``, rather than ``SparseDataFrame`` (:issue:`9850`)
+- Bug in ``SparseSeries.value_counts()`` does not count ``fill_value`` (:issue:`6749`)
 - Bug in ``SparseArray.to_dense()`` does not preserve ``dtype`` (:issue:`10648`)
 - Bug in ``SparseArray.to_dense()`` incorrectly handle ``fill_value`` (:issue:`12797`)
 - Bug in ``pd.concat()`` of ``SparseSeries`` results in dense (:issue:`10536`)
@@ -536,6 +537,9 @@ Bug Fixes
 
 
 - Bug in ``value_counts`` when ``normalize=True`` and ``dropna=True`` where nulls still contributed to the normalized count (:issue:`12558`)
+- Bug in ``Series.value_counts()`` loses name if its dtype is category (:issue:`12835`)
+- Bug in ``Series.value_counts()`` loses timezone info (:issue:`12835`)
+- Bug in ``Series.value_counts(normalize=True)`` with ``Categorical`` raises ``UnboundLocalError`` (:issue:`12835`)
 - Bug in ``Panel.fillna()`` ignoring ``inplace=True`` (:issue:`12633`)
 - Bug in ``read_csv`` when specifying ``names``, ``usecols``, and ``parse_dates`` simultaneously with the C engine (:issue:`9755`)
 - Bug in ``read_csv`` when specifying ``delim_whitespace=True`` and ``lineterminator`` simultaneously with the C engine (:issue:`12912`)

pandas/core/algorithms.py

@@ -10,6 +10,7 @@
 import pandas.core.common as com
 import pandas.algos as algos
 import pandas.hashtable as htable
+from pandas.types import api as gt
 from pandas.compat import string_types
 from pandas.tslib import iNaT
 
@@ -253,84 +254,101 @@ def value_counts(values, sort=True, ascending=False, normalize=False,
 
     """
     from pandas.core.series import Series
-    from pandas.tools.tile import cut
-    from pandas import Index, PeriodIndex, DatetimeIndex
-
     name = getattr(values, 'name', None)
-    values = Series(values).values
 
     if bins is not None:
         try:
+            from pandas.tools.tile import cut
+            values = Series(values).values
             cat, bins = cut(values, bins, retbins=True)
         except TypeError:
             raise TypeError("bins argument only works with numeric data.")
         values = cat.codes
 
-    if com.is_categorical_dtype(values.dtype):
-        result = values.value_counts(dropna)
-
+    if com.is_extension_type(values) and not com.is_datetimetz(values):
+        # handle Categorical and sparse,
+        # datetime tz can be handeled in ndarray path
+        result = Series(values).values.value_counts(dropna=dropna)
+        result.name = name
+        counts = result.values
     else:
+        # ndarray path. pass original to handle DatetimeTzBlock
+        keys, counts = _value_counts_arraylike(values, dropna=dropna)
 
-        dtype = values.dtype
-        is_period = com.is_period_arraylike(values)
-        is_datetimetz = com.is_datetimetz(values)
+        from pandas import Index, Series
+        if not isinstance(keys, Index):
+            keys = Index(keys)
+        result = Series(counts, index=keys, name=name)
 
-        if com.is_datetime_or_timedelta_dtype(dtype) or is_period or \
-                is_datetimetz:
+    if bins is not None:
+        # TODO: This next line should be more efficient
+        result = result.reindex(np.arange(len(cat.categories)),
+                                fill_value=0)
+        result.index = bins[:-1]
 
-            if is_period:
-                values = PeriodIndex(values)
-            elif is_datetimetz:
-                tz = getattr(values, 'tz', None)
-                values = DatetimeIndex(values).tz_localize(None)
+    if sort:
+        result = result.sort_values(ascending=ascending)
 
-            values = values.view(np.int64)
-            keys, counts = htable.value_count_scalar64(values, dropna)
+    if normalize:
+        result = result / float(counts.sum())
 
-            if dropna:
-                msk = keys != iNaT
-                keys, counts = keys[msk], counts[msk]
+    return result
 
-            # localize to the original tz if necessary
-            if is_datetimetz:
-                keys = DatetimeIndex(keys).tz_localize(tz)
 
-            # convert the keys back to the dtype we came in
-            else:
-                keys = keys.astype(dtype)
+def _value_counts_arraylike(values, dropna=True):
+    is_datetimetz = com.is_datetimetz(values)
+    is_period = (isinstance(values, gt.ABCPeriodIndex) or
+                 com.is_period_arraylike(values))
 
-        elif com.is_integer_dtype(dtype):
-            values = com._ensure_int64(values)
-            keys, counts = htable.value_count_scalar64(values, dropna)
-        elif com.is_float_dtype(dtype):
-            values = com._ensure_float64(values)
-            keys, counts = htable.value_count_scalar64(values, dropna)
+    orig = values
 
-        else:
-            values = com._ensure_object(values)
-            mask = com.isnull(values)
-            keys, counts = htable.value_count_object(values, mask)
-            if not dropna and mask.any():
-                keys = np.insert(keys, 0, np.NaN)
-                counts = np.insert(counts, 0, mask.sum())
+    from pandas.core.series import Series
+    values = Series(values).values
+    dtype = values.dtype
 
-        if not isinstance(keys, Index):
-            keys = Index(keys)
-        result = Series(counts, index=keys, name=name)
+    if com.is_datetime_or_timedelta_dtype(dtype) or is_period:
+        from pandas.tseries.index import DatetimeIndex
+        from pandas.tseries.period import PeriodIndex
 
-        if bins is not None:
-            # TODO: This next line should be more efficient
-            result = result.reindex(np.arange(len(cat.categories)),
-                                    fill_value=0)
-            result.index = bins[:-1]
+        if is_period:
+            values = PeriodIndex(values)
+            freq = values.freq
 
-    if sort:
-        result = result.sort_values(ascending=ascending)
+        values = values.view(np.int64)
+        keys, counts = htable.value_count_scalar64(values, dropna)
 
-    if normalize:
-        result = result / float(counts.sum())
+        if dropna:
+            msk = keys != iNaT
+            keys, counts = keys[msk], counts[msk]
 
-    return result
+        # convert the keys back to the dtype we came in
+        keys = keys.astype(dtype)
+
+        # dtype handling
+        if is_datetimetz:
+            if isinstance(orig, gt.ABCDatetimeIndex):
+                tz = orig.tz
+            else:
+                tz = orig.dt.tz
+            keys = DatetimeIndex._simple_new(keys, tz=tz)
+        if is_period:
+            keys = PeriodIndex._simple_new(keys, freq=freq)
+
+    elif com.is_integer_dtype(dtype):
+        values = com._ensure_int64(values)
+        keys, counts = htable.value_count_scalar64(values, dropna)
+    elif com.is_float_dtype(dtype):
+        values = com._ensure_float64(values)
+        keys, counts = htable.value_count_scalar64(values, dropna)
+    else:
+        values = com._ensure_object(values)
+        mask = com.isnull(values)
+        keys, counts = htable.value_count_object(values, mask)
+        if not dropna and mask.any():
+            keys = np.insert(keys, 0, np.NaN)
+            counts = np.insert(counts, 0, mask.sum())
+
+    return keys, counts
 
 
 def mode(values):

pandas/core/base.py

@@ -10,6 +10,7 @@
 from pandas.util.decorators import (Appender, cache_readonly,
                                     deprecate_kwarg, Substitution)
 from pandas.core.common import AbstractMethodError
+from pandas.types import api as gt
 from pandas.formats.printing import pprint_thing
 
 _shared_docs = dict()
@@ -291,15 +292,15 @@ def name(self):
 
     @property
     def _selection_list(self):
-        if not isinstance(self._selection, (list, tuple, com.ABCSeries,
-                                            com.ABCIndex, np.ndarray)):
+        if not isinstance(self._selection, (list, tuple, gt.ABCSeries,
+                                            gt.ABCIndex, np.ndarray)):
             return [self._selection]
         return self._selection
 
     @cache_readonly
     def _selected_obj(self):
 
-        if self._selection is None or isinstance(self.obj, com.ABCSeries):
+        if self._selection is None or isinstance(self.obj, gt.ABCSeries):
             return self.obj
         else:
             return self.obj[self._selection]
@@ -311,7 +312,7 @@ def ndim(self):
     @cache_readonly
     def _obj_with_exclusions(self):
         if self._selection is not None and isinstance(self.obj,
-                                                      com.ABCDataFrame):
+                                                      gt.ABCDataFrame):
             return self.obj.reindex(columns=self._selection_list)
 
         if len(self.exclusions) > 0:
@@ -323,7 +324,7 @@ def __getitem__(self, key):
         if self._selection is not None:
             raise Exception('Column(s) %s already selected' % self._selection)
 
-        if isinstance(key, (list, tuple, com.ABCSeries, com.ABCIndex,
+        if isinstance(key, (list, tuple, gt.ABCSeries, gt.ABCIndex,
                             np.ndarray)):
             if len(self.obj.columns.intersection(key)) != len(key):
                 bad_keys = list(set(key).difference(self.obj.columns))
@@ -551,7 +552,7 @@ def _agg(arg, func):
             if isinstance(result, list):
                 result = concat(result, keys=keys, axis=1)
             elif isinstance(list(compat.itervalues(result))[0],
-                            com.ABCDataFrame):
+                            gt.ABCDataFrame):
                 result = concat([result[k] for k in keys], keys=keys, axis=1)
             else:
                 from pandas import DataFrame
@@ -940,17 +941,8 @@ def value_counts(self, normalize=False, sort=True, ascending=False,
         counts : Series
         """
         from pandas.core.algorithms import value_counts
-        from pandas.tseries.api import DatetimeIndex, PeriodIndex
         result = value_counts(self, sort=sort, ascending=ascending,
                               normalize=normalize, bins=bins, dropna=dropna)
-
-        if isinstance(self, PeriodIndex):
-            # preserve freq
-            result.index = self._simple_new(result.index.values,
-                                            freq=self.freq)
-        elif isinstance(self, DatetimeIndex):
-            result.index = self._simple_new(result.index.values,
-                                            tz=getattr(self, 'tz', None))
         return result
 
     def unique(self):

pandas/sparse/array.py

@@ -7,6 +7,7 @@
 from numpy import nan, ndarray
 import numpy as np
 
+import pandas as pd
 from pandas.core.base import PandasObject
 import pandas.core.common as com
 
@@ -16,6 +17,7 @@
 from pandas._sparse import SparseIndex, BlockIndex, IntIndex
 import pandas._sparse as splib
 import pandas.index as _index
+import pandas.core.algorithms as algos
 import pandas.core.ops as ops
 import pandas.formats.printing as printing
 from pandas.util.decorators import Appender
@@ -503,6 +505,42 @@ def mean(self, axis=None, dtype=None, out=None):
             nsparse = self.sp_index.ngaps
             return (sp_sum + self.fill_value * nsparse) / (ct + nsparse)
 
+    def value_counts(self, dropna=True):
+        """
+        Returns a Series containing counts of unique values.
+
+        Parameters
+        ----------
+        dropna : boolean, default True
+            Don't include counts of NaN, even if NaN is in sp_values.
+
+        Returns
+        -------
+        counts : Series
+        """
+        keys, counts = algos._value_counts_arraylike(self.sp_values,
+                                                     dropna=dropna)
+        fcounts = self.sp_index.ngaps
+        if fcounts > 0:
+            if self._null_fill_value and dropna:
+                pass
+            else:
+                if self._null_fill_value:
+                    mask = pd.isnull(keys)
+                else:
+                    mask = keys == self.fill_value
+
+                if mask.any():
+                    counts[mask] += fcounts
+                else:
+                    keys = np.insert(keys, 0, self.fill_value)
+                    counts = np.insert(counts, 0, fcounts)
+
+        if not isinstance(keys, pd.Index):
+            keys = pd.Index(keys)
+        result = pd.Series(counts, index=keys)
+        return result
+
 
 def _maybe_to_dense(obj):
     """ try to convert to dense """