ENH: bug fixes, speed enh, benchmark suite to compare with xts

Author: wesm

bench/bench_take_indexing.py

@@ -0,0 +1,52 @@
+import numpy as np
+
+from pandas import *
+import pandas._tseries as lib
+
+from pandas import DataFrame
+import timeit
+
+setup = """
+from pandas import Series
+import pandas._tseries as lib
+import random
+import numpy as np
+
+import random
+n = %d
+k = %d
+arr = np.random.randn(n, k)
+indexer = np.arange(n, dtype=np.int32)
+indexer = indexer[::-1]
+"""
+
+sizes = [100, 1000, 10000, 100000]
+iters = [1000, 1000, 100, 1]
+
+fancy_2d = []
+take_2d = []
+cython_2d = []
+
+n = 1000
+
+def _timeit(stmt, size, k=5, iters=1000):
+    timer =  timeit.Timer(stmt=stmt, setup=setup % (sz, k))
+    return timer.timeit(n) / n
+
+for sz, its in zip(sizes, iters):
+    print sz
+    fancy_2d.append(_timeit('arr[indexer]', sz, iters=its))
+    take_2d.append(_timeit('arr.take(indexer, axis=0)', sz, iters=its))
+    cython_2d.append(_timeit('lib.take_axis0(arr, indexer)', sz, iters=its))
+
+df = DataFrame({'fancy' : fancy_2d,
+                'take' : take_2d,
+                'cython' : cython_2d})
+
+print df
+
+from pandas.rpy.common import r
+r('mat <- matrix(rnorm(50000), nrow=10000, ncol=5)')
+r('set.seed(12345')
+r('indexer <- sample(1:10000)')
+r('mat[indexer,]')

pandas/src/reindex.pyx

@@ -376,10 +376,10 @@ def ordered_left_join_int64(ndarray[int64_t] left, ndarray[int64_t] right):
 
 @cython.wraparound(False)
 @cython.boundscheck(False)
-def ordered_left_join_put(ndarray[int64_t] left, ndarray[int64_t] right,
-                          ndarray[float64_t, ndim=2] lvalues,
-                          ndarray[float64_t, ndim=2] rvalues,
-                          ndarray[float64_t, ndim=2] out):
+def left_join_2d(ndarray[int64_t] left, ndarray[int64_t] right,
+                 ndarray[float64_t, ndim=2] lvalues,
+                 ndarray[float64_t, ndim=2] rvalues,
+                 ndarray[float64_t, ndim=2] out):
     cdef:
         Py_ssize_t i, j, k, nright, nleft, kright, kleft
         int64_t val
@@ -409,6 +409,37 @@ def ordered_left_join_put(ndarray[int64_t] left, ndarray[int64_t] right,
             for k from kleft <= k < kleft + kright:
                 out[i, k] = NaN
 
+@cython.wraparound(False)
+@cython.boundscheck(False)
+def left_join_1d(ndarray[int64_t] left, ndarray[int64_t] right,
+                 ndarray[float64_t] lvalues,
+                 ndarray[float64_t] rvalues,
+                 ndarray[float64_t, ndim=2] out):
+    cdef:
+        Py_ssize_t i, j, nright, nleft
+        int64_t val
+
+    nleft = len(lvalues)
+    nright = len(rvalues)
+
+    j = 0
+    for i from 0 <= i < nleft:
+        out[i, 0] = lvalues[i]
+
+        val = left[i]
+
+        while j < nright and right[j] < val:
+            j += 1
+
+        if j == nright:
+            out[i, 1] = NaN
+            continue
+
+        if val == right[j]:
+            out[i, 1] = rvalues[j]
+        else:
+            out[i, 1] = NaN
+
 @cython.wraparound(False)
 @cython.boundscheck(False)
 def inner_join_indexer(ndarray[int64_t] left, ndarray[int64_t] right):
@@ -528,6 +559,83 @@ def outer_join_indexer(ndarray[int64_t] left, ndarray[int64_t] right):
             lindexer[:count].copy(),
             rindexer[:count].copy())
 
+# @cython.wraparound(False)
+# @cython.boundscheck(False)
+def take_axis0(ndarray[float64_t, ndim=2] values,
+               ndarray[int32_t] indexer,
+               out=None):
+    cdef:
+        Py_ssize_t i, j, k, n, idx
+        ndarray[float64_t, ndim=2] outbuf
+
+    n = len(indexer)
+    k = values.shape[1]
+
+    if out is None:
+        outbuf = np.empty((n, k), dtype=values.dtype)
+    else:
+        outbuf = out
+
+    for i from 0 <= i < n:
+        idx = indexer[i]
+
+        if idx == -1:
+            for j from 0 <= j < k:
+                outbuf[i, j] = NaN
+        else:
+            for j from 0 <= j < k:
+                outbuf[i, j] = values[idx, j]
+
+
+@cython.wraparound(False)
+@cython.boundscheck(False)
+def take_axis1(ndarray[float64_t, ndim=2] values,
+               ndarray[int32_t] indexer,
+               out=None):
+    cdef:
+        Py_ssize_t i, j, k, n, idx
+        ndarray[float64_t, ndim=2] outbuf
+
+    n = len(indexer)
+    k = values.shape[1]
+
+    if out is None:
+        outbuf = np.empty((n, k), dtype=values.dtype)
+    else:
+        outbuf = out
+
+    for j from 0 <= j < k:
+        idx = indexer[j]
+
+        if idx == -1:
+            for i from 0 <= i < n:
+                outbuf[i, j] = NaN
+        else:
+            for i from 0 <= i < n:
+                outbuf[i, j] = values[i, idx]
+
+@cython.wraparound(False)
+@cython.boundscheck(False)
+def take_1d(ndarray[float64_t] values, ndarray[int32_t] indexer,
+            out=None):
+    cdef:
+        Py_ssize_t i, n, idx
+        ndarray[float64_t] outbuf
+
+    n = len(indexer)
+
+    if out is None:
+        outbuf = np.empty(n, dtype=values.dtype)
+    else:
+        outbuf = out
+
+    for i from 0 <= i < n:
+        idx = indexer[i]
+        if idx == -1:
+            outbuf[i] = NaN
+        else:
+            outbuf[i] = values[idx]
+
 def ordered_put_indexer(ndarray[int64_t] left, ndarray[int64_t] right,
                         ndarray[float64_t, ndim=2] lvalues,
                         ndarray[float64_t, ndim=2] rvalues,

scripts/bench_join.R

@@ -0,0 +1,33 @@
+library(xts)
+
+iterations <- 100
+
+ns = c(100, 1000, 10000, 100000, 1000000)
+kinds = c("outer", "left", "inner")
+
+result = matrix(0, nrow=3, ncol=length(ns))
+n <- 100000
+pct.overlap <- 0.2
+
+k <- 5
+
+for (ni in 1:length(ns)){
+  n <- ns[ni]
+  rng1 <- 1:n
+  offset <- as.integer(n * pct.overlap)
+  rng2 <- rng1 + offset
+  x <- xts(matrix(rnorm(n * k), nrow=n, ncol=k),
+           as.POSIXct(Sys.Date()) + rng1)
+  y <- xts(matrix(rnorm(n * k), nrow=n, ncol=k),
+           as.POSIXct(Sys.Date()) + rng2)
+  for (i in 1:3) {
+      kind = kinds[i]
+      timing <- system.time(for (j in 1:iterations) merge(x, y, join=kind),
+                            gcFirst=F)
+      timing <- as.list(timing)
+      result[i, ni] = (timing$elapsed / iterations) * 1000
+    }
+}
+
+rownames(result) <- kinds
+colnames(result) <- log10(ns)