ENH: enable to_datetime to be vectorized, handle NAs, close #858

Author: wesm

pandas/core/datetools.py

@@ -5,7 +5,8 @@
 import pandas._tseries as lib
 import re
 
-from pandas._tseries import Timestamp, monthrange
+from pandas._tseries import Timestamp
+import pandas.core.common as com
 
 try:
     import dateutil
@@ -65,15 +66,33 @@ def parser(x):
     data = p_ufunc(arr)
     return np.array(data, dtype='M8[us]')
 
-def to_datetime(arg):
-    """Attempts to convert arg to datetime"""
+def to_datetime(arg, errors='ignore'):
+    """
+    Convert argument to datetime
+
+    Parameters
+    ----------
+    arg : string, datetime, array of strings (with possible NAs)
+    errors : {'ignore', 'raise'}, default 'ignore'
+        Errors are ignored by default (values left untouched)
+
+    Returns
+    -------
+    ret : datetime if parsing succeeded
+    """
     if arg is None:
         return arg
     elif isinstance(arg, datetime):
         return arg
+    elif isinstance(arg, np.ndarray):
+        return lib.string_to_datetime(com._ensure_object(arg),
+                                      raise_=errors == 'raise')
+
     try:
         return parser.parse(arg)
     except Exception:
+        if errors == 'raise':
+            raise
         return arg
 
 
@@ -1151,7 +1170,7 @@ class MonthEnd(DateOffset, CacheableOffset):
 
     def apply(self, other):
         n = self.n
-        _, days_in_month = monthrange(other.year, other.month)
+        _, days_in_month = lib.monthrange(other.year, other.month)
         if other.day != days_in_month:
             other = other + relativedelta(months=-1, day=31)
             if n <= 0:
@@ -1161,8 +1180,8 @@ def apply(self, other):
 
     @classmethod
     def onOffset(cls, someDate):
-        __junk, days_in_month = monthrange(someDate.year,
-                                                   someDate.month)
+        __junk, days_in_month = lib.monthrange(someDate.year,
+                                               someDate.month)
         return someDate.day == days_in_month
 
     def rule_code(self):
@@ -1184,7 +1203,7 @@ def apply(self, other):
 
     @classmethod
     def onOffset(cls, someDate):
-        firstDay, _ = monthrange(someDate.year, someDate.month)
+        firstDay, _ = lib.monthrange(someDate.year, someDate.month)
         return someDate.day == (firstDay + 1)
 
     def rule_code(self):
@@ -1202,7 +1221,7 @@ def isAnchored(self):
     def apply(self, other):
         n = self.n
 
-        wkday, days_in_month = monthrange(other.year, other.month)
+        wkday, days_in_month = lib.monthrange(other.year, other.month)
         lastBDay = days_in_month - max(((wkday + days_in_month - 1) % 7) - 4, 0)
 
         if n > 0 and not other.day >= lastBDay:
@@ -1227,15 +1246,15 @@ class BMonthBegin(DateOffset, CacheableOffset):
     def apply(self, other):
         n = self.n
 
-        wkday, _ = monthrange(other.year, other.month)
+        wkday, _ = lib.monthrange(other.year, other.month)
         firstBDay = _get_firstbday(wkday)
 
         if other.day > firstBDay and n<=0:
             # as if rolled forward already
             n += 1
 
         other = other + relativedelta(months=n)
-        wkday, _ = monthrange(other.year, other.month)
+        wkday, _ = lib.monthrange(other.year, other.month)
         firstBDay = _get_firstbday(wkday)
         result = datetime(other.year, other.month, firstBDay)
         return result
@@ -1403,7 +1422,7 @@ def isAnchored(self):
     def apply(self, other):
         n = self.n
 
-        wkday, days_in_month = monthrange(other.year, other.month)
+        wkday, days_in_month = lib.monthrange(other.year, other.month)
         lastBDay = days_in_month - max(((wkday + days_in_month - 1) % 7) - 4, 0)
 
         monthsToGo = 3 - ((other.month - self.startingMonth) % 3)
@@ -1465,7 +1484,7 @@ def apply(self, other):
         if self._normalizeFirst:
             other = normalize_date(other)
 
-        wkday, _ = monthrange(other.year, other.month)
+        wkday, _ = lib.monthrange(other.year, other.month)
 
         firstBDay = _get_firstbday(wkday)
 
@@ -1485,7 +1504,7 @@ def apply(self, other):
 
         # get the first bday for result
         other = other + relativedelta(months=3*n - monthsSince)
-        wkday, _ = monthrange(other.year, other.month)
+        wkday, _ = lib.monthrange(other.year, other.month)
         firstBDay = _get_firstbday(wkday)
         result = datetime(other.year, other.month, firstBDay)
         return result
@@ -1517,7 +1536,7 @@ def isAnchored(self):
     def apply(self, other):
         n = self.n
 
-        wkday, days_in_month = monthrange(other.year, other.month)
+        wkday, days_in_month = lib.monthrange(other.year, other.month)
 
         monthsToGo = 3 - ((other.month - self.startingMonth) % 3)
         if monthsToGo == 3:
@@ -1556,7 +1575,7 @@ def isAnchored(self):
     def apply(self, other):
         n = self.n
 
-        wkday, days_in_month = monthrange(other.year, other.month)
+        wkday, days_in_month = lib.monthrange(other.year, other.month)
 
         monthsSince = (other.month - self.startingMonth) % 3
 
@@ -1598,7 +1617,7 @@ def apply(self, other):
         if self._normalizeFirst:
             other = normalize_date(other)
 
-        wkday, days_in_month = monthrange(other.year, self.month)
+        wkday, days_in_month = lib.monthrange(other.year, self.month)
         lastBDay = (days_in_month -
                     max(((wkday + days_in_month - 1) % 7) - 4, 0))
 
@@ -1614,7 +1633,7 @@ def apply(self, other):
 
         other = other + relativedelta(years=years)
 
-        _, days_in_month = monthrange(other.year, self.month)
+        _, days_in_month = lib.monthrange(other.year, self.month)
         result = datetime(other.year, self.month, days_in_month)
 
         if result.weekday() > 4:
@@ -1646,7 +1665,7 @@ def apply(self, other):
         if self._normalizeFirst:
             other = normalize_date(other)
 
-        wkday, days_in_month = monthrange(other.year, self.month)
+        wkday, days_in_month = lib.monthrange(other.year, self.month)
 
         firstBDay = _get_firstbday(wkday)
 
@@ -1664,7 +1683,7 @@ def apply(self, other):
 
         # set first bday for result
         other = other + relativedelta(years = years)
-        wkday, days_in_month = monthrange(other.year, self.month)
+        wkday, days_in_month = lib.monthrange(other.year, self.month)
         firstBDay = _get_firstbday(wkday)
         result = datetime(other.year, self.month, firstBDay)
         return result
@@ -1688,7 +1707,7 @@ def __init__(self, n=1, **kwds):
 
     def apply(self, other):
         n = self.n
-        wkday, days_in_month = monthrange(other.year, self.month)
+        wkday, days_in_month = lib.monthrange(other.year, self.month)
         if other.month != self.month or other.day != days_in_month:
             other = datetime(other.year - 1, self.month, days_in_month)
             if n <= 0:
@@ -1697,7 +1716,7 @@ def apply(self, other):
         return other
 
     def onOffset(self, someDate):
-        wkday, days_in_month = monthrange(someDate.year, self.month)
+        wkday, days_in_month = lib.monthrange(someDate.year, self.month)
         return self.month == someDate.month and someDate.day == days_in_month
 
     def rule_code(self):
@@ -1826,6 +1845,20 @@ def rule_code(self):
 _offset_map = {
     "WEEKDAY"  : BDay(1),
 
+    # Annual - Calendar
+    "A-JAN" : YearEnd(month=1),
+    "A-FEB" : YearEnd(month=2),
+    "A-MAR" : YearEnd(month=3),
+    "A-APR" : YearEnd(month=4),
+    "A-MAY" : YearEnd(month=5),
+    "A-JUN" : YearEnd(month=6),
+    "A-JUL" : YearEnd(month=7),
+    "A-AUG" : YearEnd(month=8),
+    "A-SEP" : YearEnd(month=9),
+    "A-OCT" : YearEnd(month=10),
+    "A-NOV" : YearEnd(month=11),
+    "A-DEC" : YearEnd(month=12),
+
     # Annual - Calendar
     "A@JAN" : YearEnd(month=1),
     "A@FEB" : YearEnd(month=2),

pandas/core/index.py

@@ -1162,9 +1162,9 @@ class DatetimeIndex(Int64Index):
     _outer_indexer = _join_i8_wrapper(lib.outer_join_indexer_int64)
     _left_indexer  = _join_i8_wrapper(lib.left_join_indexer_int64,
                                       with_indexers=False)
-    _groupby       = lib.groupby_arrays # _wrap_i8_function(lib.groupby_int64)
+    _groupby = lib.groupby_arrays # _wrap_i8_function(lib.groupby_int64)
 
-    _arrmap        = _wrap_dt_function(lib.arrmap_object)
+    _arrmap = _wrap_dt_function(lib.arrmap_object)
 
     __eq__ = _dt_index_cmp('__eq__')
     __ne__ = _dt_index_cmp('__ne__')

pandas/src/tseries.pyx

@@ -568,6 +568,27 @@ def vec_binop(ndarray[object] left, ndarray[object] right, object op):
 
     return maybe_convert_bool(result)
 
+def string_to_datetime(ndarray[object] strings, raise_=False):
+    cdef:
+        Py_ssize_t i, n = len(strings)
+        object val
+    from dateutil.parser import parse
+
+    result = np.empty(n, dtype=object)
+
+    for i in range(n):
+        val = strings[i]
+        if util._checknull(val):
+            result[i] = val
+        else:
+            try:
+                result[i] = parse(val)
+            except Exception:
+                if raise_:
+                    raise
+                result[i] = val
+
+    return result
 
 def value_count_int64(ndarray[int64_t] values):
     cdef:

pandas/tests/test_timeseries.py

@@ -1,9 +1,6 @@
 # pylint: disable-msg=E1101,W0612
 
-from cStringIO import StringIO
-from datetime import datetime, timedelta
-import os
-import operator
+from datetime import datetime
 import unittest
 
 import nose
@@ -26,6 +23,35 @@
 import pandas.util.testing as tm
 import pandas
 
+
+import pandas._tseries as lib
+from datetime import datetime
+
+import cPickle as pickle
+
+import pandas.core.datetools as dt
+from pandas.core.index import Index, DatetimeIndex, Int64Index
+from pandas.core.frame import DataFrame
+
+import unittest
+import numpy as np
+
+from pandas import Series
+
+from numpy.random import rand
+
+from pandas.util.testing import assert_series_equal, assert_frame_equal
+
+from pandas.core.groupby import Tinterval
+from pandas.core.datetools import Minute, BDay, Timestamp
+
+import pandas.core.common as com
+
+try:
+    import pytz
+except ImportError:
+    pass
+
 class TestTimeSeriesDuplicates(unittest.TestCase):
 
     def setUp(self):
@@ -86,6 +112,33 @@ def assert_range_equal(left, right):
 
 class TestTimeSeries(unittest.TestCase):
 
+    def test_string_na_conversion(self):
+        from dateutil.parser import parse
+        from pandas.core.datetools import to_datetime
+
+        strings = np.array(['1/1/2000', '1/2/2000', np.nan,
+                            '1/4/2000, 12:34:56'], dtype=object)
+
+        expected = []
+        for val in strings:
+            if com.isnull(val):
+                expected.append(val)
+            else:
+                expected.append(parse(val))
+
+        result = lib.string_to_datetime(strings)
+        assert_almost_equal(result, expected)
+
+        result2 = to_datetime(strings)
+        assert_almost_equal(result, result2)
+
+        malformed = np.array(['1/100/2000', np.nan], dtype=object)
+        result = to_datetime(malformed)
+        assert_almost_equal(result, malformed)
+
+        self.assertRaises(ValueError, to_datetime, malformed,
+                          errors='raise')
+
     def test_dti_slicing(self):
         dti = DatetimeIndex(start='1/1/2005', end='12/1/2005', freq='M')
         dti2 = dti[[1,3,5]]
@@ -201,32 +254,6 @@ def test_pad_require_monotonicity(self):
         self.assertRaises(AssertionError, rng2.get_indexer, rng,
                           method='pad')
 
-import pandas._tseries as lib
-from datetime import datetime
-
-import cPickle as pickle
-
-import pandas.core.datetools as dt
-from pandas.core.index import Index, DatetimeIndex, Int64Index
-from pandas.core.frame import DataFrame
-
-import unittest
-import numpy as np
-
-from pandas import Series
-
-from numpy.random import rand
-
-from pandas.util.testing import assert_series_equal, assert_frame_equal
-
-from pandas.core.groupby import Tinterval
-from pandas.core.datetools import Minute, BDay, Timestamp
-
-try:
-    import pytz
-except ImportError:
-    pass
-
 def _skip_if_no_pytz():
     try:
         import pytz
@@ -900,6 +927,7 @@ def test_dayofmonthoffset(self):
                     self.assert_(t.weekday() == day)
 
 
+
 if __name__ == '__main__':
     nose.runmodule(argv=[__file__,'-vvs','-x','--pdb', '--pdb-failure'],
                    exit=False)