Merge pull request #3615 from jreback/read_csv_na_values

BUG: (GH3611) Fix read_csv to correctly encode identical na_values

Author: jreback

RELEASE.rst

@@ -118,6 +118,8 @@ pandas 0.11.1
   - Fix modulo and integer division on Series,DataFrames to act similary to ``float`` dtypes to return 
     ``np.nan`` or ``np.inf`` as appropriate (GH3590_)
   - Fix incorrect dtype on groupby with ``as_index=False`` (GH3610_)
+  - Fix ``read_csv`` to correctly encode identical na_values, e.g. ``na_values=[-999.0,-999]``
+    was failing (GH3611_)
 
 .. _GH3164: https://github.com/pydata/pandas/issues/3164
 .. _GH2786: https://github.com/pydata/pandas/issues/2786
@@ -166,6 +168,7 @@ pandas 0.11.1
 .. _GH3610: https://github.com/pydata/pandas/issues/3610
 .. _GH3596: https://github.com/pydata/pandas/issues/3596
 .. _GH3435: https://github.com/pydata/pandas/issues/3435
+.. _GH3611: https://github.com/pydata/pandas/issues/3611
 
 
 pandas 0.11.0

pandas/io/parsers.py

@@ -1715,12 +1715,14 @@ def _clean_na_values(na_values, keep_default_na=True):
     else:
         if not com.is_list_like(na_values):
             na_values = [na_values]
-        na_values = set(list(na_values))
+        na_values = set(_stringify_na_values(na_values))
         if keep_default_na:
             na_values = na_values | _NA_VALUES
 
     return na_values
 
+def _stringify_na_values(na_values):
+    return [ str(x) for x in na_values ]
 
 def _clean_index_names(columns, index_col):
     if not _is_index_col(index_col):
@@ -1771,7 +1773,7 @@ def _get_empty_meta(columns, index_col, index_names):
 def _get_na_values(col, na_values):
     if isinstance(na_values, dict):
         if col in na_values:
-            return set(list(na_values[col]))
+            return set(_stringify_na_values(list(na_values[col])))
         else:
             return _NA_VALUES
     else:

pandas/io/tests/test_parsers.py

@@ -498,6 +498,17 @@ def test_quoting(self):
         df = self.read_table(StringIO(good_line_small), sep='\t')
         self.assert_(len(df) == 3)
 
+    def test_non_string_na_values(self):
+        # GH3611, na_values that are not a string are an issue
+        with ensure_clean('__non_string_na_values__.csv') as path:
+            df = DataFrame({'A' : [-999, 2, 3], 'B' : [1.2, -999, 4.5]})
+            df.to_csv(path, sep=' ', index=False)
+            result1 = read_csv(path, sep= ' ', header=0, na_values=['-999.0','-999'])
+            result2 = read_csv(path, sep= ' ', header=0, na_values=[-999,-999.0])
+            result3 = read_csv(path, sep= ' ', header=0, na_values=[-999.0,-999])
+            tm.assert_frame_equal(result1,result2)
+            tm.assert_frame_equal(result2,result3)
+
     def test_custom_na_values(self):
         data = """A,B,C
 ignore,this,row