From 504130b3f6d767fb74fc1091671aa051c7a7732c Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Wed, 2 Jan 2019 18:24:54 +0100 Subject: [PATCH 1/2] REF: shift ravel in infer_dtype --- pandas/_libs/lib.pyx | 9 +- pandas/core/algorithms.py | 14 +- pandas/core/arrays/datetimes.py | 2 +- pandas/core/arrays/integer.py | 4 +- pandas/core/arrays/timedeltas.py | 2 +- pandas/core/dtypes/cast.py | 12 +- pandas/core/dtypes/common.py | 3 +- pandas/core/dtypes/missing.py | 2 +- pandas/core/indexes/base.py | 8 +- pandas/core/indexes/multi.py | 3 +- pandas/core/internals/construction.py | 2 +- pandas/core/reshape/merge.py | 3 +- pandas/core/reshape/tile.py | 2 +- pandas/core/series.py | 4 +- pandas/core/sorting.py | 2 +- pandas/io/parsers.py | 2 +- pandas/io/pytables.py | 12 +- pandas/io/sql.py | 23 +--- pandas/io/stata.py | 4 +- pandas/plotting/_converter.py | 2 +- pandas/tests/dtypes/test_inference.py | 164 ++++++++++++----------- pandas/tests/series/test_constructors.py | 4 +- pandas/tests/test_strings.py | 2 +- 23 files changed, 147 insertions(+), 138 deletions(-) diff --git a/pandas/_libs/lib.pyx b/pandas/_libs/lib.pyx index d6e2b9a5288f5..1124000c97875 100644 --- a/pandas/_libs/lib.pyx +++ b/pandas/_libs/lib.pyx @@ -623,7 +623,7 @@ def clean_index_list(obj: list): return obj, all_arrays # don't force numpy coerce with nan's - inferred = infer_dtype(obj) + inferred = infer_dtype(obj, skipna=False) if inferred in ['string', 'bytes', 'unicode', 'mixed', 'mixed-integer']: return np.asarray(obj, dtype=object), 0 elif inferred in ['integer']: @@ -1210,6 +1210,10 @@ def infer_dtype(value: object, skipna: bool=False) -> str: values = construct_1d_object_array_from_listlike(value) values = getattr(values, 'values', values) + + # make contiguous + values = values.ravel() + if skipna: values = values[~isnaobj(values)] @@ -1220,9 +1224,6 @@ def infer_dtype(value: object, skipna: bool=False) -> str: if values.dtype != np.object_: values = values.astype('O') - # make contiguous - values = values.ravel() - n = len(values) if n == 0: return 'empty' diff --git a/pandas/core/algorithms.py b/pandas/core/algorithms.py index 8d85b84ec7507..08453ffd01ad1 100644 --- a/pandas/core/algorithms.py +++ b/pandas/core/algorithms.py @@ -165,7 +165,7 @@ def _ensure_arraylike(values): ensure that we are arraylike if not already """ if not is_array_like(values): - inferred = lib.infer_dtype(values) + inferred = lib.infer_dtype(values, skipna=False) if inferred in ['mixed', 'string', 'unicode']: if isinstance(values, tuple): values = list(values) @@ -202,8 +202,10 @@ def _get_hashtable_algo(values): if ndtype == 'object': - # its cheaper to use a String Hash Table than Object - if lib.infer_dtype(values) in ['string']: + # it's cheaper to use a String Hash Table than Object; we infer + # including nulls because that is the only difference between + # StringHashTable and ObjectHashtable + if lib.infer_dtype(values, skipna=False) in ['string']: ndtype = 'string' else: ndtype = 'object' @@ -220,8 +222,10 @@ def _get_data_algo(values, func_map): values, dtype, ndtype = _ensure_data(values) if ndtype == 'object': - # its cheaper to use a String Hash Table than Object - if lib.infer_dtype(values) in ['string']: + # it's cheaper to use a String Hash Table than Object; we infer + # including nulls because that is the only difference between + # StringHashTable and ObjectHashtable + if lib.infer_dtype(values, skipna=False) in ['string']: ndtype = 'string' f = func_map.get(ndtype, func_map['object']) diff --git a/pandas/core/arrays/datetimes.py b/pandas/core/arrays/datetimes.py index 8b0565a36648f..ddb7d3968d1b6 100644 --- a/pandas/core/arrays/datetimes.py +++ b/pandas/core/arrays/datetimes.py @@ -1581,7 +1581,7 @@ def sequence_to_dt64ns(data, dtype=None, copy=False, # TODO: We do not have tests specific to string-dtypes, # also complex or categorical or other extension copy = False - if lib.infer_dtype(data) == 'integer': + if lib.infer_dtype(data, skipna=False) == 'integer': data = data.astype(np.int64) else: # data comes back here as either i8 to denote UTC timestamps diff --git a/pandas/core/arrays/integer.py b/pandas/core/arrays/integer.py index eaec76b96a24d..af2c05bbee7c2 100644 --- a/pandas/core/arrays/integer.py +++ b/pandas/core/arrays/integer.py @@ -171,8 +171,8 @@ def coerce_to_array(values, dtype, mask=None, copy=False): values = np.array(values, copy=copy) if is_object_dtype(values): - inferred_type = lib.infer_dtype(values) - if inferred_type is 'mixed' and isna(values).all(): + inferred_type = lib.infer_dtype(values, skipna=True) + if inferred_type == 'empty': values = np.empty(len(values)) values.fill(np.nan) elif inferred_type not in ['floating', 'integer', diff --git a/pandas/core/arrays/timedeltas.py b/pandas/core/arrays/timedeltas.py index 78570be8dc07f..975419b8fc8fd 100644 --- a/pandas/core/arrays/timedeltas.py +++ b/pandas/core/arrays/timedeltas.py @@ -561,7 +561,7 @@ def __floordiv__(self, other): elif is_object_dtype(other): result = [self[n] // other[n] for n in range(len(self))] result = np.array(result) - if lib.infer_dtype(result) == 'timedelta': + if lib.infer_dtype(result, skipna=False) == 'timedelta': result, _ = sequence_to_td64ns(result) return type(self)(result) return result diff --git a/pandas/core/dtypes/cast.py b/pandas/core/dtypes/cast.py index eae9eb97f35fe..04ef62ffc6f7c 100644 --- a/pandas/core/dtypes/cast.py +++ b/pandas/core/dtypes/cast.py @@ -73,7 +73,8 @@ def trans(x): if isinstance(dtype, string_types): if dtype == 'infer': - inferred_type = lib.infer_dtype(ensure_object(result.ravel())) + inferred_type = lib.infer_dtype(ensure_object(result.ravel()), + skipna=False) if inferred_type == 'boolean': dtype = 'bool' elif inferred_type == 'integer': @@ -458,7 +459,7 @@ def infer_dtype_from_array(arr, pandas_dtype=False): return arr.dtype, np.asarray(arr) # don't force numpy coerce with nan's - inferred = lib.infer_dtype(arr) + inferred = lib.infer_dtype(arr, skipna=False) if inferred in ['string', 'bytes', 'unicode', 'mixed', 'mixed-integer']: return (np.object_, arr) @@ -937,10 +938,11 @@ def try_timedelta(v): # We have at least a NaT and a string # try timedelta first to avoid spurious datetime conversions - # e.g. '00:00:01' is a timedelta but - # technically is also a datetime + # e.g. '00:00:01' is a timedelta but technically is also a datetime value = try_timedelta(v) - if lib.infer_dtype(value) in ['mixed']: + if lib.infer_dtype(value, skipna=False) in ['mixed']: + # cannot skip missing values, as NaT implies that the string + # is actually a datetime value = try_datetime(v) return value diff --git a/pandas/core/dtypes/common.py b/pandas/core/dtypes/common.py index e1141c6b6b3a8..ddf310383cc7b 100644 --- a/pandas/core/dtypes/common.py +++ b/pandas/core/dtypes/common.py @@ -704,7 +704,8 @@ def is_datetime_arraylike(arr): if isinstance(arr, ABCDatetimeIndex): return True elif isinstance(arr, (np.ndarray, ABCSeries)): - return arr.dtype == object and lib.infer_dtype(arr) == 'datetime' + return (is_object_dtype(arr.dtype) + and lib.infer_dtype(arr, skipna=False) == 'datetime') return getattr(arr, 'inferred_type', None) == 'datetime' diff --git a/pandas/core/dtypes/missing.py b/pandas/core/dtypes/missing.py index 21ec14ace3e44..b22cb1050f140 100644 --- a/pandas/core/dtypes/missing.py +++ b/pandas/core/dtypes/missing.py @@ -474,7 +474,7 @@ def _infer_fill_value(val): if is_datetimelike(val): return np.array('NaT', dtype=val.dtype) elif is_object_dtype(val.dtype): - dtype = lib.infer_dtype(ensure_object(val)) + dtype = lib.infer_dtype(ensure_object(val), skipna=False) if dtype in ['datetime', 'datetime64']: return np.array('NaT', dtype=_NS_DTYPE) elif dtype in ['timedelta', 'timedelta64']: diff --git a/pandas/core/indexes/base.py b/pandas/core/indexes/base.py index 1380c5caed1c9..acabe14365cd1 100644 --- a/pandas/core/indexes/base.py +++ b/pandas/core/indexes/base.py @@ -347,7 +347,7 @@ def __new__(cls, data=None, dtype=None, copy=False, name=None, # should not be coerced # GH 11836 if is_integer_dtype(dtype): - inferred = lib.infer_dtype(data) + inferred = lib.infer_dtype(data, skipna=False) if inferred == 'integer': data = maybe_cast_to_integer_array(data, dtype, copy=copy) @@ -377,7 +377,7 @@ def __new__(cls, data=None, dtype=None, copy=False, name=None, else: data = data.astype(dtype) elif is_float_dtype(dtype): - inferred = lib.infer_dtype(data) + inferred = lib.infer_dtype(data, skipna=False) if inferred == 'string': pass else: @@ -415,7 +415,7 @@ def __new__(cls, data=None, dtype=None, copy=False, name=None, subarr = subarr.copy() if dtype is None: - inferred = lib.infer_dtype(subarr) + inferred = lib.infer_dtype(subarr, skipna=False) if inferred == 'integer': try: return cls._try_convert_to_int_index( @@ -1720,7 +1720,7 @@ def inferred_type(self): """ Return a string of the type inferred from the values. """ - return lib.infer_dtype(self) + return lib.infer_dtype(self, skipna=False) @cache_readonly def is_all_dates(self): diff --git a/pandas/core/indexes/multi.py b/pandas/core/indexes/multi.py index 60059d5a43440..14b09e26ae661 100644 --- a/pandas/core/indexes/multi.py +++ b/pandas/core/indexes/multi.py @@ -2318,7 +2318,8 @@ def _partial_tup_index(self, tup, side='left'): section = labs[start:end] if lab not in lev: - if not lev.is_type_compatible(lib.infer_dtype([lab])): + if not lev.is_type_compatible(lib.infer_dtype([lab], + skipna=False)): raise TypeError('Level type mismatch: %s' % lab) # short circuit diff --git a/pandas/core/internals/construction.py b/pandas/core/internals/construction.py index b18b966406bbb..67aa7101220ca 100644 --- a/pandas/core/internals/construction.py +++ b/pandas/core/internals/construction.py @@ -667,7 +667,7 @@ def sanitize_array(data, index, dtype=None, copy=False, subarr = np.array(data, dtype=object, copy=copy) if is_object_dtype(subarr.dtype) and dtype != 'object': - inferred = lib.infer_dtype(subarr) + inferred = lib.infer_dtype(subarr, skipna=False) if inferred == 'period': try: subarr = period_array(subarr) diff --git a/pandas/core/reshape/merge.py b/pandas/core/reshape/merge.py index 58344c0ec9ec7..b459600354290 100644 --- a/pandas/core/reshape/merge.py +++ b/pandas/core/reshape/merge.py @@ -943,7 +943,8 @@ def _maybe_coerce_merge_keys(self): 'representation', UserWarning) # let's infer and see if we are ok - elif lib.infer_dtype(lk) == lib.infer_dtype(rk): + elif (lib.infer_dtype(lk, skipna=False) + == lib.infer_dtype(rk, skipna=False)): pass # Check if we are trying to merge on obviously diff --git a/pandas/core/reshape/tile.py b/pandas/core/reshape/tile.py index 21a93f7deec8b..6f95b14993228 100644 --- a/pandas/core/reshape/tile.py +++ b/pandas/core/reshape/tile.py @@ -416,7 +416,7 @@ def _convert_bin_to_numeric_type(bins, dtype): ------ ValueError if bins are not of a compat dtype to dtype """ - bins_dtype = infer_dtype(bins) + bins_dtype = infer_dtype(bins, skipna=False) if is_timedelta64_dtype(dtype): if bins_dtype in ['timedelta', 'timedelta64']: bins = to_timedelta(bins).view(np.int64) diff --git a/pandas/core/series.py b/pandas/core/series.py index 3637081e09f8c..52b60339a7d68 100644 --- a/pandas/core/series.py +++ b/pandas/core/series.py @@ -875,7 +875,7 @@ def _get_with(self, key): if isinstance(key, Index): key_type = key.inferred_type else: - key_type = lib.infer_dtype(key) + key_type = lib.infer_dtype(key, skipna=False) if key_type == 'integer': if self.index.is_integer() or self.index.is_floating(): @@ -1012,7 +1012,7 @@ def _set_with(self, key, value): if isinstance(key, Index): key_type = key.inferred_type else: - key_type = lib.infer_dtype(key) + key_type = lib.infer_dtype(key, skipna=False) if key_type == 'integer': if self.index.inferred_type == 'integer': diff --git a/pandas/core/sorting.py b/pandas/core/sorting.py index b34dfddcc66e1..ef69939d6e978 100644 --- a/pandas/core/sorting.py +++ b/pandas/core/sorting.py @@ -454,7 +454,7 @@ def sort_mixed(values): return np.concatenate([nums, np.asarray(strs, dtype=object)]) sorter = None - if PY3 and lib.infer_dtype(values) == 'mixed-integer': + if PY3 and lib.infer_dtype(values, skipna=False) == 'mixed-integer': # unorderable in py3 if mixed str/int ordered = sort_mixed(values) else: diff --git a/pandas/io/parsers.py b/pandas/io/parsers.py index 2861f32e54e5e..5590e8f445c67 100755 --- a/pandas/io/parsers.py +++ b/pandas/io/parsers.py @@ -1300,7 +1300,7 @@ def _validate_usecols_arg(usecols): elif not is_list_like(usecols): raise ValueError(msg) else: - usecols_dtype = lib.infer_dtype(usecols) + usecols_dtype = lib.infer_dtype(usecols, skipna=False) if usecols_dtype not in ('empty', 'integer', 'string', 'unicode'): raise ValueError(msg) diff --git a/pandas/io/pytables.py b/pandas/io/pytables.py index a894b8788f8d8..183e5fb18e874 100644 --- a/pandas/io/pytables.py +++ b/pandas/io/pytables.py @@ -1952,7 +1952,7 @@ def set_atom(self, block, block_items, existing_col, min_itemsize, return self.set_atom_complex(block) dtype = block.dtype.name - inferred_type = lib.infer_dtype(block.values) + inferred_type = lib.infer_dtype(block.values, skipna=False) if inferred_type == 'date': raise TypeError( @@ -1998,7 +1998,7 @@ def set_atom_string(self, block, block_items, existing_col, min_itemsize, data = block.values # see if we have a valid string type - inferred_type = lib.infer_dtype(data.ravel()) + inferred_type = lib.infer_dtype(data.ravel(), skipna=False) if inferred_type != 'string': # we cannot serialize this data, so report an exception on a column @@ -2006,7 +2006,7 @@ def set_atom_string(self, block, block_items, existing_col, min_itemsize, for i, item in enumerate(block_items): col = block.iget(i) - inferred_type = lib.infer_dtype(col.ravel()) + inferred_type = lib.infer_dtype(col.ravel(), skipna=False) if inferred_type != 'string': raise TypeError( "Cannot serialize the column [%s] because\n" @@ -2744,7 +2744,7 @@ def write_array(self, key, value, items=None): # infer the type, warn if we have a non-string type here (for # performance) - inferred_type = lib.infer_dtype(value.ravel()) + inferred_type = lib.infer_dtype(value.ravel(), skipna=False) if empty_array: pass elif inferred_type == 'string': @@ -4511,7 +4511,7 @@ def _convert_index(index, encoding=None, errors='strict', format_type=None): if isinstance(index, MultiIndex): raise TypeError('MultiIndex not supported here!') - inferred_type = lib.infer_dtype(index) + inferred_type = lib.infer_dtype(index, skipna=False) values = np.asarray(index) @@ -4744,7 +4744,7 @@ def __init__(self, table, where=None, start=None, stop=None): # see if we have a passed coordinate like try: - inferred = lib.infer_dtype(where) + inferred = lib.infer_dtype(where, skipna=False) if inferred == 'integer' or inferred == 'boolean': where = np.asarray(where) if where.dtype == np.bool_: diff --git a/pandas/io/sql.py b/pandas/io/sql.py index 0eefa85211194..2f4093e154a95 100644 --- a/pandas/io/sql.py +++ b/pandas/io/sql.py @@ -857,27 +857,15 @@ def _harmonize_columns(self, parse_dates=None): except KeyError: pass # this column not in results - def _get_notna_col_dtype(self, col): - """ - Infer datatype of the Series col. In case the dtype of col is 'object' - and it contains NA values, this infers the datatype of the not-NA - values. Needed for inserting typed data containing NULLs, GH8778. - """ - col_for_inference = col - if col.dtype == 'object': - notnadata = col[~isna(col)] - if len(notnadata): - col_for_inference = notnadata - - return lib.infer_dtype(col_for_inference) - def _sqlalchemy_type(self, col): dtype = self.dtype or {} if col.name in dtype: return self.dtype[col.name] - col_type = self._get_notna_col_dtype(col) + # Infer type of column, while ignoring missing values. + # Needed for inserting typed data containing NULLs, GH 8778. + col_type = lib.infer_dtype(col, skipna=True) from sqlalchemy.types import (BigInteger, Integer, Float, Text, Boolean, @@ -1374,7 +1362,10 @@ def _sql_type_name(self, col): if col.name in dtype: return dtype[col.name] - col_type = self._get_notna_col_dtype(col) + # Infer type of column, while ignoring missing values. + # Needed for inserting typed data containing NULLs, GH 8778. + col_type = lib.infer_dtype(col, skipna=True) + if col_type == 'timedelta64': warnings.warn("the 'timedelta' type is not supported, and will be " "written as integer values (ns frequency) to the " diff --git a/pandas/io/stata.py b/pandas/io/stata.py index fcd99e7cdce0d..aabc66058b626 100644 --- a/pandas/io/stata.py +++ b/pandas/io/stata.py @@ -396,7 +396,7 @@ def parse_dates_safe(dates, delta=False, year=False, days=False): to_datetime(d['year'], format='%Y').astype(np.int64)) d['days'] = days // NS_PER_DAY - elif infer_dtype(dates) == 'datetime': + elif infer_dtype(dates, skipna=False) == 'datetime': if delta: delta = dates.values - stata_epoch f = lambda x: \ @@ -1867,7 +1867,7 @@ def _dtype_to_default_stata_fmt(dtype, column, dta_version=114, if force_strl: return '%9s' if dtype.type == np.object_: - inferred_dtype = infer_dtype(column.dropna()) + inferred_dtype = infer_dtype(column, skipna=False) if not (inferred_dtype in ('string', 'unicode') or len(column) == 0): raise ValueError('Column `{col}` cannot be exported.\n\nOnly ' diff --git a/pandas/plotting/_converter.py b/pandas/plotting/_converter.py index 8cab00fba3aa8..4c6b3b5132fec 100644 --- a/pandas/plotting/_converter.py +++ b/pandas/plotting/_converter.py @@ -246,7 +246,7 @@ def _convert_1d(values, units, axis): return values.asfreq(axis.freq)._ndarray_values elif isinstance(values, Index): return values.map(lambda x: get_datevalue(x, axis.freq)) - elif lib.infer_dtype(values) == 'period': + elif lib.infer_dtype(values, skipna=False) == 'period': # https://github.com/pandas-dev/pandas/issues/24304 # convert ndarray[period] -> PeriodIndex return PeriodIndex(values, freq=axis.freq)._ndarray_values diff --git a/pandas/tests/dtypes/test_inference.py b/pandas/tests/dtypes/test_inference.py index d9b1b0db90562..fff91991ee251 100644 --- a/pandas/tests/dtypes/test_inference.py +++ b/pandas/tests/dtypes/test_inference.py @@ -342,11 +342,11 @@ def test_infer_dtype_bytes(self): # string array of bytes arr = np.array(list('abc'), dtype='S1') - assert lib.infer_dtype(arr) == compare + assert lib.infer_dtype(arr, skipna=False) == compare # object array of bytes arr = arr.astype(object) - assert lib.infer_dtype(arr) == compare + assert lib.infer_dtype(arr, skipna=False) == compare # object array of bytes with missing values assert lib.infer_dtype([b'a', np.nan, b'c'], skipna=True) == compare @@ -530,87 +530,91 @@ def test_inferred_dtype_fixture(self, any_skipna_inferred_dtype): # make sure the inferred dtype of the fixture is as requested assert inferred_dtype == lib.infer_dtype(values, skipna=True) - def test_length_zero(self): - result = lib.infer_dtype(np.array([], dtype='i4')) + @pytest.mark.parametrize('skipna', [True, False]) + def test_length_zero(self, skipna): + result = lib.infer_dtype(np.array([], dtype='i4'), skipna=skipna) assert result == 'integer' - result = lib.infer_dtype([]) + result = lib.infer_dtype([], skipna=skipna) assert result == 'empty' # GH 18004 arr = np.array([np.array([], dtype=object), np.array([], dtype=object)]) - result = lib.infer_dtype(arr) + result = lib.infer_dtype(arr, skipna=skipna) assert result == 'empty' def test_integers(self): arr = np.array([1, 2, 3, np.int64(4), np.int32(5)], dtype='O') - result = lib.infer_dtype(arr) + result = lib.infer_dtype(arr, skipna=False) assert result == 'integer' arr = np.array([1, 2, 3, np.int64(4), np.int32(5), 'foo'], dtype='O') - result = lib.infer_dtype(arr) + result = lib.infer_dtype(arr, skipna=False) assert result == 'mixed-integer' arr = np.array([1, 2, 3, 4, 5], dtype='i4') - result = lib.infer_dtype(arr) + result = lib.infer_dtype(arr, skipna=False) assert result == 'integer' def test_bools(self): arr = np.array([True, False, True, True, True], dtype='O') - result = lib.infer_dtype(arr) + result = lib.infer_dtype(arr, skipna=False) assert result == 'boolean' arr = np.array([np.bool_(True), np.bool_(False)], dtype='O') - result = lib.infer_dtype(arr) + result = lib.infer_dtype(arr, skipna=False) assert result == 'boolean' arr = np.array([True, False, True, 'foo'], dtype='O') - result = lib.infer_dtype(arr) + result = lib.infer_dtype(arr, skipna=False) assert result == 'mixed' arr = np.array([True, False, True], dtype=bool) - result = lib.infer_dtype(arr) + result = lib.infer_dtype(arr, skipna=False) assert result == 'boolean' arr = np.array([True, np.nan, False], dtype='O') result = lib.infer_dtype(arr, skipna=True) assert result == 'boolean' + result = lib.infer_dtype(arr, skipna=False) + assert result == 'mixed' + def test_floats(self): arr = np.array([1., 2., 3., np.float64(4), np.float32(5)], dtype='O') - result = lib.infer_dtype(arr) + result = lib.infer_dtype(arr, skipna=False) assert result == 'floating' arr = np.array([1, 2, 3, np.float64(4), np.float32(5), 'foo'], dtype='O') - result = lib.infer_dtype(arr) + result = lib.infer_dtype(arr, skipna=False) assert result == 'mixed-integer' arr = np.array([1, 2, 3, 4, 5], dtype='f4') - result = lib.infer_dtype(arr) + result = lib.infer_dtype(arr, skipna=False) assert result == 'floating' arr = np.array([1, 2, 3, 4, 5], dtype='f8') - result = lib.infer_dtype(arr) + result = lib.infer_dtype(arr, skipna=False) assert result == 'floating' def test_decimals(self): # GH15690 arr = np.array([Decimal(1), Decimal(2), Decimal(3)]) - result = lib.infer_dtype(arr) + result = lib.infer_dtype(arr, skipna=False) assert result == 'decimal' arr = np.array([1.0, 2.0, Decimal(3)]) - result = lib.infer_dtype(arr) + result = lib.infer_dtype(arr, skipna=False) assert result == 'mixed' arr = np.array([Decimal(1), Decimal('NaN'), Decimal(3)]) - result = lib.infer_dtype(arr) + result = lib.infer_dtype(arr, skipna=False) assert result == 'decimal' arr = np.array([Decimal(1), np.nan, Decimal(3)], dtype='O') - result = lib.infer_dtype(arr) + result = lib.infer_dtype(arr, skipna=False) assert result == 'decimal' def test_string(self): @@ -618,7 +622,7 @@ def test_string(self): def test_unicode(self): arr = [u'a', np.nan, u'c'] - result = lib.infer_dtype(arr) + result = lib.infer_dtype(arr, skipna=False) assert result == 'mixed' arr = [u'a', np.nan, u'c'] @@ -652,135 +656,135 @@ def test_infer_dtype_datetime(self): arr = np.array([Timestamp('2011-01-01'), Timestamp('2011-01-02')]) - assert lib.infer_dtype(arr) == 'datetime' + assert lib.infer_dtype(arr, skipna=False) == 'datetime' arr = np.array([np.datetime64('2011-01-01'), np.datetime64('2011-01-01')], dtype=object) - assert lib.infer_dtype(arr) == 'datetime64' + assert lib.infer_dtype(arr, skipna=False) == 'datetime64' arr = np.array([datetime(2011, 1, 1), datetime(2012, 2, 1)]) - assert lib.infer_dtype(arr) == 'datetime' + assert lib.infer_dtype(arr, skipna=False) == 'datetime' # starts with nan for n in [pd.NaT, np.nan]: arr = np.array([n, pd.Timestamp('2011-01-02')]) - assert lib.infer_dtype(arr) == 'datetime' + assert lib.infer_dtype(arr, skipna=False) == 'datetime' arr = np.array([n, np.datetime64('2011-01-02')]) - assert lib.infer_dtype(arr) == 'datetime64' + assert lib.infer_dtype(arr, skipna=False) == 'datetime64' arr = np.array([n, datetime(2011, 1, 1)]) - assert lib.infer_dtype(arr) == 'datetime' + assert lib.infer_dtype(arr, skipna=False) == 'datetime' arr = np.array([n, pd.Timestamp('2011-01-02'), n]) - assert lib.infer_dtype(arr) == 'datetime' + assert lib.infer_dtype(arr, skipna=False) == 'datetime' arr = np.array([n, np.datetime64('2011-01-02'), n]) - assert lib.infer_dtype(arr) == 'datetime64' + assert lib.infer_dtype(arr, skipna=False) == 'datetime64' arr = np.array([n, datetime(2011, 1, 1), n]) - assert lib.infer_dtype(arr) == 'datetime' + assert lib.infer_dtype(arr, skipna=False) == 'datetime' # different type of nat arr = np.array([np.timedelta64('nat'), np.datetime64('2011-01-02')], dtype=object) - assert lib.infer_dtype(arr) == 'mixed' + assert lib.infer_dtype(arr, skipna=False) == 'mixed' arr = np.array([np.datetime64('2011-01-02'), np.timedelta64('nat')], dtype=object) - assert lib.infer_dtype(arr) == 'mixed' + assert lib.infer_dtype(arr, skipna=False) == 'mixed' # mixed datetime arr = np.array([datetime(2011, 1, 1), pd.Timestamp('2011-01-02')]) - assert lib.infer_dtype(arr) == 'datetime' + assert lib.infer_dtype(arr, skipna=False) == 'datetime' # should be datetime? arr = np.array([np.datetime64('2011-01-01'), pd.Timestamp('2011-01-02')]) - assert lib.infer_dtype(arr) == 'mixed' + assert lib.infer_dtype(arr, skipna=False) == 'mixed' arr = np.array([pd.Timestamp('2011-01-02'), np.datetime64('2011-01-01')]) - assert lib.infer_dtype(arr) == 'mixed' + assert lib.infer_dtype(arr, skipna=False) == 'mixed' arr = np.array([np.nan, pd.Timestamp('2011-01-02'), 1]) - assert lib.infer_dtype(arr) == 'mixed-integer' + assert lib.infer_dtype(arr, skipna=False) == 'mixed-integer' arr = np.array([np.nan, pd.Timestamp('2011-01-02'), 1.1]) - assert lib.infer_dtype(arr) == 'mixed' + assert lib.infer_dtype(arr, skipna=False) == 'mixed' arr = np.array([np.nan, '2011-01-01', pd.Timestamp('2011-01-02')]) - assert lib.infer_dtype(arr) == 'mixed' + assert lib.infer_dtype(arr, skipna=False) == 'mixed' def test_infer_dtype_timedelta(self): arr = np.array([pd.Timedelta('1 days'), pd.Timedelta('2 days')]) - assert lib.infer_dtype(arr) == 'timedelta' + assert lib.infer_dtype(arr, skipna=False) == 'timedelta' arr = np.array([np.timedelta64(1, 'D'), np.timedelta64(2, 'D')], dtype=object) - assert lib.infer_dtype(arr) == 'timedelta' + assert lib.infer_dtype(arr, skipna=False) == 'timedelta' arr = np.array([timedelta(1), timedelta(2)]) - assert lib.infer_dtype(arr) == 'timedelta' + assert lib.infer_dtype(arr, skipna=False) == 'timedelta' # starts with nan for n in [pd.NaT, np.nan]: arr = np.array([n, Timedelta('1 days')]) - assert lib.infer_dtype(arr) == 'timedelta' + assert lib.infer_dtype(arr, skipna=False) == 'timedelta' arr = np.array([n, np.timedelta64(1, 'D')]) - assert lib.infer_dtype(arr) == 'timedelta' + assert lib.infer_dtype(arr, skipna=False) == 'timedelta' arr = np.array([n, timedelta(1)]) - assert lib.infer_dtype(arr) == 'timedelta' + assert lib.infer_dtype(arr, skipna=False) == 'timedelta' arr = np.array([n, pd.Timedelta('1 days'), n]) - assert lib.infer_dtype(arr) == 'timedelta' + assert lib.infer_dtype(arr, skipna=False) == 'timedelta' arr = np.array([n, np.timedelta64(1, 'D'), n]) - assert lib.infer_dtype(arr) == 'timedelta' + assert lib.infer_dtype(arr, skipna=False) == 'timedelta' arr = np.array([n, timedelta(1), n]) - assert lib.infer_dtype(arr) == 'timedelta' + assert lib.infer_dtype(arr, skipna=False) == 'timedelta' # different type of nat arr = np.array([np.datetime64('nat'), np.timedelta64(1, 'D')], dtype=object) - assert lib.infer_dtype(arr) == 'mixed' + assert lib.infer_dtype(arr, skipna=False) == 'mixed' arr = np.array([np.timedelta64(1, 'D'), np.datetime64('nat')], dtype=object) - assert lib.infer_dtype(arr) == 'mixed' + assert lib.infer_dtype(arr, skipna=False) == 'mixed' def test_infer_dtype_period(self): # GH 13664 arr = np.array([pd.Period('2011-01', freq='D'), pd.Period('2011-02', freq='D')]) - assert lib.infer_dtype(arr) == 'period' + assert lib.infer_dtype(arr, skipna=False) == 'period' arr = np.array([pd.Period('2011-01', freq='D'), pd.Period('2011-02', freq='M')]) - assert lib.infer_dtype(arr) == 'period' + assert lib.infer_dtype(arr, skipna=False) == 'period' # starts with nan for n in [pd.NaT, np.nan]: arr = np.array([n, pd.Period('2011-01', freq='D')]) - assert lib.infer_dtype(arr) == 'period' + assert lib.infer_dtype(arr, skipna=False) == 'period' arr = np.array([n, pd.Period('2011-01', freq='D'), n]) - assert lib.infer_dtype(arr) == 'period' + assert lib.infer_dtype(arr, skipna=False) == 'period' # different type of nat arr = np.array([np.datetime64('nat'), pd.Period('2011-01', freq='M')], dtype=object) - assert lib.infer_dtype(arr) == 'mixed' + assert lib.infer_dtype(arr, skipna=False) == 'mixed' arr = np.array([pd.Period('2011-01', freq='M'), np.datetime64('nat')], dtype=object) - assert lib.infer_dtype(arr) == 'mixed' + assert lib.infer_dtype(arr, skipna=False) == 'mixed' @pytest.mark.parametrize( "data", @@ -850,60 +854,62 @@ def test_infer_datetimelike_array_nan_nat_like(self, first, second, def test_infer_dtype_all_nan_nat_like(self): arr = np.array([np.nan, np.nan]) - assert lib.infer_dtype(arr) == 'floating' + assert lib.infer_dtype(arr, skipna=False) == 'floating' # nan and None mix are result in mixed arr = np.array([np.nan, np.nan, None]) - assert lib.infer_dtype(arr) == 'mixed' + assert lib.infer_dtype(arr, skipna=True) == 'empty' + assert lib.infer_dtype(arr, skipna=False) == 'mixed' arr = np.array([None, np.nan, np.nan]) - assert lib.infer_dtype(arr) == 'mixed' + assert lib.infer_dtype(arr, skipna=True) == 'empty' + assert lib.infer_dtype(arr, skipna=False) == 'mixed' # pd.NaT arr = np.array([pd.NaT]) - assert lib.infer_dtype(arr) == 'datetime' + assert lib.infer_dtype(arr, skipna=False) == 'datetime' arr = np.array([pd.NaT, np.nan]) - assert lib.infer_dtype(arr) == 'datetime' + assert lib.infer_dtype(arr, skipna=False) == 'datetime' arr = np.array([np.nan, pd.NaT]) - assert lib.infer_dtype(arr) == 'datetime' + assert lib.infer_dtype(arr, skipna=False) == 'datetime' arr = np.array([np.nan, pd.NaT, np.nan]) - assert lib.infer_dtype(arr) == 'datetime' + assert lib.infer_dtype(arr, skipna=False) == 'datetime' arr = np.array([None, pd.NaT, None]) - assert lib.infer_dtype(arr) == 'datetime' + assert lib.infer_dtype(arr, skipna=False) == 'datetime' # np.datetime64(nat) arr = np.array([np.datetime64('nat')]) - assert lib.infer_dtype(arr) == 'datetime64' + assert lib.infer_dtype(arr, skipna=False) == 'datetime64' for n in [np.nan, pd.NaT, None]: arr = np.array([n, np.datetime64('nat'), n]) - assert lib.infer_dtype(arr) == 'datetime64' + assert lib.infer_dtype(arr, skipna=False) == 'datetime64' arr = np.array([pd.NaT, n, np.datetime64('nat'), n]) - assert lib.infer_dtype(arr) == 'datetime64' + assert lib.infer_dtype(arr, skipna=False) == 'datetime64' arr = np.array([np.timedelta64('nat')], dtype=object) - assert lib.infer_dtype(arr) == 'timedelta' + assert lib.infer_dtype(arr, skipna=False) == 'timedelta' for n in [np.nan, pd.NaT, None]: arr = np.array([n, np.timedelta64('nat'), n]) - assert lib.infer_dtype(arr) == 'timedelta' + assert lib.infer_dtype(arr, skipna=False) == 'timedelta' arr = np.array([pd.NaT, n, np.timedelta64('nat'), n]) - assert lib.infer_dtype(arr) == 'timedelta' + assert lib.infer_dtype(arr, skipna=False) == 'timedelta' # datetime / timedelta mixed arr = np.array([pd.NaT, np.datetime64('nat'), np.timedelta64('nat'), np.nan]) - assert lib.infer_dtype(arr) == 'mixed' + assert lib.infer_dtype(arr, skipna=False) == 'mixed' arr = np.array([np.timedelta64('nat'), np.datetime64('nat')], dtype=object) - assert lib.infer_dtype(arr) == 'mixed' + assert lib.infer_dtype(arr, skipna=False) == 'mixed' def test_is_datetimelike_array_all_nan_nat_like(self): arr = np.array([np.nan, pd.NaT, np.datetime64('nat')]) @@ -967,7 +973,7 @@ def test_date(self): assert index.inferred_type == 'date' dates = [date(2012, 1, day) for day in range(1, 20)] + [np.nan] - result = lib.infer_dtype(dates) + result = lib.infer_dtype(dates, skipna=False) assert result == 'mixed' result = lib.infer_dtype(dates, skipna=True) @@ -1011,8 +1017,10 @@ def test_object(self): # GH 7431 # cannot infer more than this as only a single element arr = np.array([None], dtype='O') - result = lib.infer_dtype(arr) + result = lib.infer_dtype(arr, skipna=False) assert result == 'mixed' + result = lib.infer_dtype(arr, skipna=True) + assert result == 'empty' def test_to_object_array_width(self): # see gh-13320 @@ -1043,17 +1051,17 @@ def test_categorical(self): # GH 8974 from pandas import Categorical, Series arr = Categorical(list('abc')) - result = lib.infer_dtype(arr) + result = lib.infer_dtype(arr, skipna=False) assert result == 'categorical' - result = lib.infer_dtype(Series(arr)) + result = lib.infer_dtype(Series(arr), skipna=False) assert result == 'categorical' arr = Categorical(list('abc'), categories=['cegfab'], ordered=True) - result = lib.infer_dtype(arr) + result = lib.infer_dtype(arr, skipna=False) assert result == 'categorical' - result = lib.infer_dtype(Series(arr)) + result = lib.infer_dtype(Series(arr), skipna=False) assert result == 'categorical' diff --git a/pandas/tests/series/test_constructors.py b/pandas/tests/series/test_constructors.py index f5a445e2cca9a..a9f78096f3cd1 100644 --- a/pandas/tests/series/test_constructors.py +++ b/pandas/tests/series/test_constructors.py @@ -806,12 +806,12 @@ def test_constructor_with_datetime_tz(self): s = Series([pd.Timestamp('2013-01-01 13:00:00-0800', tz='US/Pacific'), pd.Timestamp('2013-01-02 14:00:00-0800', tz='US/Pacific')]) assert s.dtype == 'datetime64[ns, US/Pacific]' - assert lib.infer_dtype(s) == 'datetime64' + assert lib.infer_dtype(s, skipna=False) == 'datetime64' s = Series([pd.Timestamp('2013-01-01 13:00:00-0800', tz='US/Pacific'), pd.Timestamp('2013-01-02 14:00:00-0800', tz='US/Eastern')]) assert s.dtype == 'object' - assert lib.infer_dtype(s) == 'datetime' + assert lib.infer_dtype(s, skipna=False) == 'datetime' # with all NaT s = Series(pd.NaT, index=[0, 1], dtype='datetime64[ns, US/Eastern]') diff --git a/pandas/tests/test_strings.py b/pandas/tests/test_strings.py index d4ea21632edf9..7cea3be03d1a7 100644 --- a/pandas/tests/test_strings.py +++ b/pandas/tests/test_strings.py @@ -155,7 +155,7 @@ def any_allowed_skipna_inferred_dtype(request): >>> import pandas._libs.lib as lib >>> >>> def test_something(any_allowed_skipna_inferred_dtype): - ... inferred_dtype, values = any_skipna_inferred_dtype + ... inferred_dtype, values = any_allowed_skipna_inferred_dtype ... # will pass ... assert lib.infer_dtype(values, skipna=True) == inferred_dtype """ From 5fb02b8433602e68efc445d63ddf9bbb38166c4d Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Wed, 2 Jan 2019 19:10:47 +0100 Subject: [PATCH 2/2] Fix oversight --- pandas/io/stata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/io/stata.py b/pandas/io/stata.py index aabc66058b626..aad57fc489fb6 100644 --- a/pandas/io/stata.py +++ b/pandas/io/stata.py @@ -1867,7 +1867,7 @@ def _dtype_to_default_stata_fmt(dtype, column, dta_version=114, if force_strl: return '%9s' if dtype.type == np.object_: - inferred_dtype = infer_dtype(column, skipna=False) + inferred_dtype = infer_dtype(column, skipna=True) if not (inferred_dtype in ('string', 'unicode') or len(column) == 0): raise ValueError('Column `{col}` cannot be exported.\n\nOnly '