window skipNaN (#996)

* skipNaN

* generalize non-strict reducers (#997)

* generalize non-strict reducers

* tweak

* don’t coerce for mode

* non-strict reducers

* DRY

* mode can be undefined, not NaN

* fix first/last defined logic

* don’t coerce for first and last

* numbers cannot be null

* shorter

* fix comment

Co-authored-by: Philippe Rivière <[email protected]>

* Update README

* Update README

* don’t coerce for min and max

Co-authored-by: Mike Bostock <[email protected]>

Author: Fil

README.md

@@ -1890,9 +1890,9 @@ The Plot.windowX and Plot.windowY transforms compute a moving window around each
 * **k** - the window size (the number of elements in the window)
 * **anchor** - how to align the window: *start*, *middle*, or *end*
 * **reduce** - the aggregation method (window reducer)
-* **strict** - if true, disallow window truncation; defaults to false
+* **strict** - if true, output undefined if any window value is undefined; defaults to false
 
-If the **strict** option is true, the resulting start values or end values or both (depending on the **anchor**) of each series may be undefined since there are not enough elements to create a window of size **k**. If the **strict** option is false (the default), the window will be automatically truncated as needed. For example, if **k** is 24 and **anchor** is *middle*, then the initial 11 values have effective window sizes of 13, 14, 15, … 23, and likewise the last 12 values have effective window sizes of 23, 22, 21, … 12. Values computed with a truncated window may be noiser; if you would prefer to not show this data, set the **strict** option to true.
+If the **strict** option is true, the output start values or end values or both (depending on the **anchor**) of each series may be undefined since there are not enough elements to create a window of size **k**; output values may also be undefined if some of the input values in the corresponding window are undefined. If the **strict** option is false (the default), the window will be automatically truncated as needed, and undefined input values are ignored. For example, if **k** is 24 and **anchor** is *middle*, then the initial 11 values have effective window sizes of 13, 14, 15, … 23, and likewise the last 12 values have effective window sizes of 23, 22, 21, … 12. Values computed with a truncated window may be noisy; if you would prefer to not show this data, set the **strict** option to true.
 
 The following window reducers are supported:
 

src/transforms/window.js

@@ -1,7 +1,8 @@
-import {mapX, mapY} from "./map.js";
 import {deviation, max, min, median, mode, variance} from "d3";
+import {defined} from "../defined.js";
+import {percentile, take} from "../options.js";
 import {warn} from "../warnings.js";
-import {percentile} from "../options.js";
+import {mapX, mapY} from "./map.js";
 
 export function windowX(windowOptions = {}, options) {
   if (arguments.length === 1) options = windowOptions;
@@ -21,9 +22,7 @@ export function window(options = {}) {
     warn(`Warning: the shift option is deprecated; please use anchor "${anchor}" instead.`);
   }
   if (!((k = Math.floor(k)) > 0)) throw new Error(`invalid k: ${k}`);
-  const r = maybeReduce(reduce);
-  const s = maybeAnchor(anchor, k);
-  return (strict ? r : looseReducer(r))(k, s);
+  return maybeReduce(reduce)(k, maybeAnchor(anchor, k), strict);
 }
 
 function maybeAnchor(anchor = "middle", k) {
@@ -46,52 +45,36 @@ function maybeShift(shift) {
 
 function maybeReduce(reduce = "mean") {
   if (typeof reduce === "string") {
-    if (/^p\d{2}$/i.test(reduce)) return reduceSubarray(percentile(reduce));
+    if (/^p\d{2}$/i.test(reduce)) return reduceNumbers(percentile(reduce));
     switch (reduce.toLowerCase()) {
-      case "deviation": return reduceSubarray(deviation);
-      case "max": return reduceSubarray(max);
+      case "deviation": return reduceNumbers(deviation);
+      case "max": return reduceArray(max);
       case "mean": return reduceMean;
-      case "median": return reduceSubarray(median);
-      case "min": return reduceSubarray(min);
-      case "mode": return reduceSubarray(mode);
+      case "median": return reduceNumbers(median);
+      case "min": return reduceArray(min);
+      case "mode": return reduceArray(mode);
       case "sum": return reduceSum;
-      case "variance": return reduceSubarray(variance);
+      case "variance": return reduceNumbers(variance);
       case "difference": return reduceDifference;
       case "ratio": return reduceRatio;
       case "first": return reduceFirst;
       case "last": return reduceLast;
     }
   }
   if (typeof reduce !== "function") throw new Error(`invalid reduce: ${reduce}`);
-  return reduceSubarray(reduce);
-}
-
-function looseReducer(reducer) {
-  return (k, s) => {
-    const reduce = reducer(k, s);
-    return {
-      map(I, S, T) {
-        const n = I.length;
-        reduce.map(I, S, T);
-        for (let i = 0; i < s; ++i) {
-          const j = Math.min(n, i + k - s);
-          reducer(j, i).map(slice(I, 0, j), S, T);
-        }
-        for (let i = n - k + s + 1; i < n; ++i) {
-          const j = Math.max(0, i - s);
-          reducer(n - j, i - j).map(slice(I, j, n), S, T);
-        }
-      }
-    };
-  };
+  return reduceArray(reduce);
 }
 
 function slice(I, i, j) {
   return I.subarray ? I.subarray(i, j) : I.slice(i, j);
 }
 
-function reduceSubarray(f) {
-  return (k, s) => ({
+// Note that the subarray may include NaN in the non-strict case; we expect the
+// function f to handle that itself (e.g., by filtering as needed). The D3
+// reducers (e.g., min, max, mean, median) do, and it’s faster to avoid
+// redundant filtering.
+function reduceNumbers(f) {
+  return (k, s, strict) => strict ? ({
     map(I, S, T) {
       const C = Float64Array.from(I, i => S[i] === null ? NaN : S[i]);
       let nans = 0;
@@ -102,11 +85,44 @@ function reduceSubarray(f) {
         if (isNaN(C[i])) --nans;
       }
     }
+  }) : ({
+    map(I, S, T) {
+      const C = Float64Array.from(I, i => S[i] === null ? NaN : S[i]);
+      for (let i = -s; i < 0; ++i) {
+        T[I[i + s]] = f(C.subarray(0, i + k));
+      }
+      for (let i = 0, n = I.length - s; i < n; ++i) {
+        T[I[i + s]] = f(C.subarray(i, i + k));
+      }
+    }
+  });
+}
+
+function reduceArray(f) {
+  return (k, s, strict) => strict ? ({
+    map(I, S, T) {
+      let count = 0;
+      for (let i = 0; i < k - 1; ++i) count += defined(S[I[i]]);
+      for (let i = 0, n = I.length - k + 1; i < n; ++i) {
+        count += defined(S[I[i + k - 1]]);
+        if (count === k) T[I[i + s]] = f(take(S, slice(I, i, i + k)));
+        count -= defined(S[I[i]]);
+      }
+    }
+  }) : ({
+    map(I, S, T) {
+      for (let i = -s; i < 0; ++i) {
+        T[I[i + s]] = f(take(S, slice(I, 0, i + k)));
+      }
+      for (let i = 0, n = I.length - s; i < n; ++i) {
+        T[I[i + s]] = f(take(S, slice(I, i, i + k)));
+      }
+    }
   });
 }
 
-function reduceSum(k, s) {
-  return {
+function reduceSum(k, s, strict) {
+  return strict ? ({
     map(I, S, T) {
       let nans = 0;
       let sum = 0;
@@ -125,61 +141,147 @@ function reduceSum(k, s) {
         else sum -= +a;
       }
     }
-  };
-}
-
-function reduceMean(k, s) {
-  const sum = reduceSum(k, s);
-  return {
+  }) : ({
     map(I, S, T) {
-      sum.map(I, S, T);
-      for (let i = 0, n = I.length - k + 1; i < n; ++i) {
-        T[I[i + s]] /= k;
+      let sum = 0;
+      const n = I.length;
+      for (let i = 0, j = Math.min(n, k - s - 1); i < j; ++i) {
+        sum += +S[I[i]] || 0;
+      }
+      for (let i = -s, j = n - s; i < j; ++i) {
+        sum += +S[I[i + k - 1]] || 0;
+        T[I[i + s]] = sum;
+        sum -= +S[I[i]] || 0;
       }
     }
-  };
+  });
+}
+
+function reduceMean(k, s, strict) {
+  if (strict) {
+    const sum = reduceSum(k, s, strict);
+    return {
+      map(I, S, T) {
+        sum.map(I, S, T);
+        for (let i = 0, n = I.length - k + 1; i < n; ++i) {
+          T[I[i + s]] /= k;
+        }
+      }
+    };
+  } else {
+    return {
+      map(I, S, T) {
+        let sum = 0;
+        let count = 0;
+        const n = I.length;
+        for (let i = 0, j = Math.min(n, k - s - 1); i < j; ++i) {
+          let v = S[I[i]];
+          if (v !== null && !isNaN(v = +v)) sum += v, ++count;
+        }
+        for (let i = -s, j = n - s; i < j; ++i) {
+          let a = S[I[i + k - 1]];
+          let b = S[I[i]];
+          if (a !== null && !isNaN(a = +a)) sum += a, ++count;
+          T[I[i + s]] = sum / count;
+          if (b !== null && !isNaN(b = +b)) sum -= b, --count;
+        }
+      }
+    };
+  }
+}
+
+function firstDefined(S, I, i, k) {
+  for (let j = i + k; i < j; ++i) {
+    const v = S[I[i]];
+    if (defined(v)) return v;
+  }
+}
+
+function lastDefined(S, I, i, k) {
+  for (let j = i + k - 1; j >= i; --j) {
+    const v = S[I[j]];
+    if (defined(v)) return v;
+  }
+}
+
+function firstNumber(S, I, i, k) {
+  for (let j = i + k; i < j; ++i) {
+    let v = S[I[i]];
+    if (v !== null && !isNaN(v = +v)) return v;
+  }
 }
 
-function reduceDifference(k, s) {
-  return {
+function lastNumber(S, I, i, k) {
+  for (let j = i + k - 1; j >= i; --j) {
+    let v = S[I[j]];
+    if (v !== null && !isNaN(v = +v)) return v;
+  }
+}
+
+function reduceDifference(k, s, strict) {
+  return strict ? ({
     map(I, S, T) {
       for (let i = 0, n = I.length - k; i < n; ++i) {
         const a = S[I[i]];
         const b = S[I[i + k - 1]];
         T[I[i + s]] = a === null || b === null ? NaN : b - a;
       }
     }
-  };
+  }) : ({
+    map(I, S, T) {
+      for (let i = -s, n = I.length - k + s + 1; i < n; ++i) {
+        T[I[i + s]] = lastNumber(S, I, i, k) - firstNumber(S, I, i, k);
+      }
+    }
+  });
 }
 
-function reduceRatio(k, s) {
-  return {
+function reduceRatio(k, s, strict) {
+  return strict ? ({
     map(I, S, T) {
       for (let i = 0, n = I.length - k; i < n; ++i) {
         const a = S[I[i]];
         const b = S[I[i + k - 1]];
         T[I[i + s]] = a === null || b === null ? NaN : b / a;
       }
     }
-  };
+  }) : ({
+    map(I, S, T) {
+      for (let i = -s, n = I.length - k + s + 1; i < n; ++i) {
+        T[I[i + s]] = lastNumber(S, I, i, k) / firstNumber(S, I, i, k);
+      }
+    }
+  });
 }
 
-function reduceFirst(k, s) {
-  return {
+function reduceFirst(k, s, strict) {
+  return strict ? ({
     map(I, S, T) {
       for (let i = 0, n = I.length - k; i < n; ++i) {
         T[I[i + s]] = S[I[i]];
       }
     }
-  };
+  }) : ({
+    map(I, S, T) {
+      for (let i = -s, n = I.length - k + s + 1; i < n; ++i) {
+        T[I[i + s]] = firstDefined(S, I, i, k);
+      }
+    }
+  });
 }
 
-function reduceLast(k, s) {
-  return {
+function reduceLast(k, s, strict) {
+  return strict ? ({
     map(I, S, T) {
       for (let i = 0, n = I.length - k; i < n; ++i) {
         T[I[i + s]] = S[I[i + k - 1]];
       }
     }
-  };
+  }) : ({
+    map(I, S, T) {
+      for (let i = -s, n = I.length - k + s + 1; i < n; ++i) {
+        T[I[i + s]] = lastDefined(S, I, i, k);
+      }
+    }
+  });
 }