|
| 1 | +"""Function registry, aliasing and fill-value logic shared by every implementation. |
| 2 | +
|
| 3 | +This module must not import numpy - the pure python implementation is loaded |
| 4 | +before numpy is known to be available, and has to work without it. |
| 5 | +""" |
| 6 | + |
| 7 | +nan = float("nan") # nan is just this float |
| 8 | + |
| 9 | + |
| 10 | +aggregate_common_doc = """ |
| 11 | + See readme file at https://github.com/ml31415/numpy-groupies for a full |
| 12 | + description. Below we reproduce the "Full description of inputs" |
| 13 | + section from that readme, note that the text below makes references to |
| 14 | + other portions of the readme that are not shown here. |
| 15 | +
|
| 16 | + group_idx: |
| 17 | + this is an array of non-negative integers, to be used as the "labels" |
| 18 | + with which to group the values in ``a``. Although we have so far |
| 19 | + assumed that ``group_idx`` is one-dimensional, and the same length as |
| 20 | + ``a``, it can in fact be two-dimensional (or some form of nested |
| 21 | + sequences that can be converted to 2D). When ``group_idx`` is 2D, the |
| 22 | + size of the 0th dimension corresponds to the number of dimensions in |
| 23 | + the output, i.e. ``group_idx[i,j]`` gives the index into the ith |
| 24 | + dimension in the output |
| 25 | + for ``a[j]``. Note that ``a`` should still be 1D (or scalar), with |
| 26 | + length matching ``group_idx.shape[1]``. |
| 27 | + a: |
| 28 | + this is the array of values to be aggregated. See above for a |
| 29 | + simple demonstration of what this means. ``a`` will normally be a |
| 30 | + one-dimensional array, however it can also be a scalar in some cases. |
| 31 | + func: default='sum' |
| 32 | + the function to use for aggregation. See the section above for |
| 33 | + details. Note that the simplest way to specify the function is using a |
| 34 | + string (e.g. ``func='max'``) however a number of aliases are also |
| 35 | + defined (e.g. you can use the ``func=np.max``, or even ``func=max``, |
| 36 | + where ``max`` is the |
| 37 | + builtin function). To check the available aliases see ``utils.py``. |
| 38 | + size: default=None |
| 39 | + the shape of the output array. If ``None``, the maximum value in |
| 40 | + ``group_idx`` will set the size of the output. Note that for |
| 41 | + multidimensional output you need to list the size of each dimension |
| 42 | + here, or give ``None``. |
| 43 | + fill_value: default=DEFAULT_FILL_VALUE |
| 44 | + in the example above, group 2 does not have any data, so requires some |
| 45 | + kind of filling value. By default the value is chosen per function, |
| 46 | + following what the corresponding numpy function returns for an empty |
| 47 | + slice: ``0`` for sums and counts, ``1`` for products, ``False`` for the |
| 48 | + boolean functions, ``nan`` for the averaging ones (mean, median, var, |
| 49 | + std, and min/max/first/last on floating input), ``0`` for the |
| 50 | + trapezoidal integral (as it is for a single sample), ``-1`` for |
| 51 | + argmax/argmin, and an empty sequence for array/sort. Integer output |
| 52 | + cannot hold ``nan``, so those functions fall back to ``0``. For complex |
| 53 | + output the default ``nan`` fills with a nan in *both* parts, so that an |
| 54 | + empty group looks the same in every implementation. Use |
| 55 | + ``utils.default_fill_value(func, dtype)`` to query the value, or pass |
| 56 | + your own. Note that there are some subtle interactions between what is |
| 57 | + permitted for ``fill_value`` and the input/output ``dtype`` - exceptions |
| 58 | + should be raised in most cases to alert the programmer if issues arise. |
| 59 | + order: default='C' |
| 60 | + this is relevant only for multidimensional output. It controls the |
| 61 | + layout of the output array in memory, can be ``'F'`` for fortran-style. |
| 62 | + dtype: default=None |
| 63 | + the ``dtype`` of the output. By default something sensible is chosen |
| 64 | + based on the input, aggregation function, and ``fill_value``. |
| 65 | + Complex input keeps the complex dtype wherever the result is complex |
| 66 | + (sum, prod, mean, median, trapezoid, first, last, sort, cumsum), while |
| 67 | + var, std and sumofsquares measure squared magnitudes and return a real |
| 68 | + dtype - matching numpy. Requesting a non-complex ``dtype`` for complex |
| 69 | + input is refused for every function that does return a complex result, |
| 70 | + since the imaginary part would be lost. The order-dependent functions |
| 71 | + (min, max, argmax, argmin and their ``nan`` counterparts) follow numpy's |
| 72 | + lexicographic ordering - the real part decides, the imaginary part only |
| 73 | + breaks a tie - in every implementation, including numba, which compares |
| 74 | + the two parts itself because neither python nor numba orders complex |
| 75 | + numbers. |
| 76 | + axis: default=None |
| 77 | + allows aggregation to be performed along a single axis of a |
| 78 | + multi-dimensional array ``a``. In that case ``group_idx`` must be 1D |
| 79 | + with length matching ``a.shape[axis]``, or have the same |
| 80 | + dimensionality as ``a`` with a shape broadcastable to ``a.shape`` |
| 81 | + (e.g. to use separate group labels for each row). In either case the |
| 82 | + groups are broadcast out along the remaining axes of ``a``. Not |
| 83 | + supported by the pure python implementation. |
| 84 | + reverse: default=False |
| 85 | + only relevant for ``func='sort'`` - sorts the items within each group |
| 86 | + in descending order instead of ascending. |
| 87 | + ddof: default=0 |
| 88 | + passed through into calculations of variance and standard deviation |
| 89 | + (see above). |
| 90 | + dx: default=1.0 |
| 91 | + passed through into the calculation of the trapezoidal integral |
| 92 | + ``trapezoid`` (see above), where it is the sample spacing. |
| 93 | +""" |
| 94 | + |
| 95 | +funcs_common = [ |
| 96 | + "first", |
| 97 | + "last", |
| 98 | + "len", |
| 99 | + "mean", |
| 100 | + "median", |
| 101 | + "trapezoid", |
| 102 | + "var", |
| 103 | + "std", |
| 104 | + "allnan", |
| 105 | + "anynan", |
| 106 | + "max", |
| 107 | + "min", |
| 108 | + "argmax", |
| 109 | + "argmin", |
| 110 | + "sumofsquares", |
| 111 | + "cumsum", |
| 112 | + "cumprod", |
| 113 | + "cummax", |
| 114 | + "cummin", |
| 115 | +] |
| 116 | +funcs_no_separate_nan = frozenset(["sort", "array", "allnan", "anynan"]) |
| 117 | + |
| 118 | + |
| 119 | +class _DefaultFillValue: |
| 120 | + """Sentinel for ``fill_value``, asking for the function specific default.""" |
| 121 | + |
| 122 | + def __repr__(self): |
| 123 | + return "DEFAULT_FILL_VALUE" |
| 124 | + |
| 125 | + def __reduce__(self): |
| 126 | + # keep the singleton through pickling and deepcopy |
| 127 | + return (_get_default_fill_value, ()) |
| 128 | + |
| 129 | + |
| 130 | +def _get_default_fill_value(): |
| 131 | + return DEFAULT_FILL_VALUE |
| 132 | + |
| 133 | + |
| 134 | +DEFAULT_FILL_VALUE = _DefaultFillValue() |
| 135 | + |
| 136 | +# The value an absent group gets, per aggregation function, following the |
| 137 | +# convention that it is whatever numpy returns for an empty slice - where the |
| 138 | +# output datatype can represent it (see ``resolve_fill_value``). |
| 139 | +_default_fill_values = { |
| 140 | + "sum": 0, |
| 141 | + "prod": 1, |
| 142 | + "all": False, |
| 143 | + "any": False, |
| 144 | + "allnan": False, |
| 145 | + "anynan": False, |
| 146 | + "len": 0, |
| 147 | + "mean": nan, |
| 148 | + "median": nan, |
| 149 | + # an empty domain has a zero integral, just like a single sample does |
| 150 | + "trapezoid": 0, |
| 151 | + "var": nan, |
| 152 | + "std": nan, |
| 153 | + "min": nan, |
| 154 | + "max": nan, |
| 155 | + "first": nan, |
| 156 | + "last": nan, |
| 157 | + "argmax": -1, |
| 158 | + "argmin": -1, |
| 159 | + "sumofsquares": 0, |
| 160 | + "cumsum": 0, |
| 161 | + "cumprod": 1, |
| 162 | + "cummax": 0, |
| 163 | + "cummin": 0, |
| 164 | + # 'array' and 'sort' ask for an empty sequence, which their implementations |
| 165 | + # interpret as 'leave the empty group as it is' - a fresh list per call is |
| 166 | + # handed out below so that callers cannot alias each other's results. |
| 167 | + "array": [], |
| 168 | + "sort": [], |
| 169 | +} |
| 170 | + |
| 171 | + |
| 172 | +def _fill_value_key(func): |
| 173 | + """Map a function (name, alias or callable) onto a ``_default_fill_values`` key.""" |
| 174 | + try: |
| 175 | + name = aliasing_py[func] |
| 176 | + except (KeyError, TypeError): |
| 177 | + name = getattr(func, "__name__", "") |
| 178 | + if name not in _default_fill_values and str(name).startswith("nan"): |
| 179 | + # nan-variants fill like their plain counterparts |
| 180 | + name = name[3:] |
| 181 | + return name |
| 182 | + |
| 183 | + |
| 184 | +def _is_inexact_dtype(dtype): |
| 185 | + """True for everything that can hold nan - numpy dtypes by kind, python types by identity.""" |
| 186 | + kind = getattr(dtype, "kind", None) |
| 187 | + if kind is not None: # a numpy dtype |
| 188 | + return kind in "fc" |
| 189 | + if isinstance(dtype, str): |
| 190 | + return dtype.startswith(("float", "complex")) |
| 191 | + return isinstance(dtype, type) and issubclass(dtype, (float, complex)) |
| 192 | + |
| 193 | + |
| 194 | +def _is_complex_dtype(dtype): |
| 195 | + """True for complex numpy dtypes, the complex builtin and complex dtype strings.""" |
| 196 | + kind = getattr(dtype, "kind", None) |
| 197 | + if kind is not None: # a numpy dtype |
| 198 | + return kind == "c" |
| 199 | + if isinstance(dtype, str): |
| 200 | + return dtype.startswith("complex") |
| 201 | + return isinstance(dtype, type) and issubclass(dtype, complex) |
| 202 | + |
| 203 | + |
| 204 | +def resolve_fill_value(func, fill_value, dtype): |
| 205 | + """Replace the ``DEFAULT_FILL_VALUE`` sentinel with the default of ``func``. |
| 206 | +
|
| 207 | + Anything else than the sentinel is returned untouched. ``nan`` is only |
| 208 | + handed out if ``dtype`` can represent it, otherwise the value falls back to |
| 209 | + ``0`` - changing the output datatype of e.g. an integer ``min`` would be a |
| 210 | + worse surprise than an unspectacular filling value. |
| 211 | + """ |
| 212 | + if fill_value is not DEFAULT_FILL_VALUE: |
| 213 | + return fill_value |
| 214 | + key = _fill_value_key(func) |
| 215 | + # nan only fits where the output can hold it - the averaging and |
| 216 | + # dispersion functions coerce their result to a float type anyway |
| 217 | + fits = dtype is None or key in _forced_float_types or key in _forced_real_float_types or _is_inexact_dtype(dtype) |
| 218 | + value = _default_fill_values.get(key) |
| 219 | + if value is None: |
| 220 | + # a custom callable has no function specific default |
| 221 | + value = nan if fits else 0 |
| 222 | + elif value != value and not fits: |
| 223 | + value = 0 |
| 224 | + if ( |
| 225 | + isinstance(value, float) |
| 226 | + and value != value |
| 227 | + and dtype is not None |
| 228 | + and _is_complex_dtype(dtype) |
| 229 | + and key not in _forced_real_float_types |
| 230 | + ): |
| 231 | + # a nan of a complex output has to be a nan in both parts - the numpy |
| 232 | + # kernels divide an empty sum by zero and so get nan+nanj for free, |
| 233 | + # and filling with nan+0j would make the backends disagree |
| 234 | + value = nan + nan * 1j |
| 235 | + return list(value) if isinstance(value, list) else value |
| 236 | + |
| 237 | + |
| 238 | +_alias_str = { |
| 239 | + "or": "any", |
| 240 | + "and": "all", |
| 241 | + "add": "sum", |
| 242 | + "count": "len", |
| 243 | + "plus": "sum", |
| 244 | + "multiply": "prod", |
| 245 | + "product": "prod", |
| 246 | + "times": "prod", |
| 247 | + "amax": "max", |
| 248 | + "maximum": "max", |
| 249 | + "amin": "min", |
| 250 | + "minimum": "min", |
| 251 | + "split": "array", |
| 252 | + "splice": "array", |
| 253 | + "sorted": "sort", |
| 254 | + "asort": "sort", |
| 255 | + "asorted": "sort", |
| 256 | + "rsorted": "sort", |
| 257 | + "dsort": "sort", |
| 258 | +} |
| 259 | + |
| 260 | +_alias_builtin = { |
| 261 | + all: "all", |
| 262 | + any: "any", |
| 263 | + len: "len", |
| 264 | + max: "max", |
| 265 | + min: "min", |
| 266 | + sum: "sum", |
| 267 | + sorted: "sort", |
| 268 | + slice: "array", |
| 269 | + list: "array", |
| 270 | +} |
| 271 | + |
| 272 | + |
| 273 | +def get_aliasing(*extra): |
| 274 | + """ |
| 275 | + Assembles a dictionary that maps both strings and functions to a list of supported function names. |
| 276 | +
|
| 277 | + Examples: |
| 278 | + alias['add'] = 'sum' |
| 279 | + alias[sorted] = 'sort' |
| 280 | +
|
| 281 | + This function should only be called during import. |
| 282 | + """ |
| 283 | + alias = {k: k for k in funcs_common} |
| 284 | + alias.update(_alias_str) |
| 285 | + alias.update((fn, fn) for fn in _alias_builtin.values()) |
| 286 | + alias.update(_alias_builtin) |
| 287 | + for d in extra: |
| 288 | + alias.update(d) |
| 289 | + alias.update((k, k) for k in set(alias.values())) |
| 290 | + # Treat nan-functions as firstclass member and add them directly |
| 291 | + for key in set(alias.values()): |
| 292 | + if key not in funcs_no_separate_nan and not key.startswith("nan"): |
| 293 | + key = "nan" + key |
| 294 | + alias[key] = key |
| 295 | + return alias |
| 296 | + |
| 297 | + |
| 298 | +aliasing_py = get_aliasing() |
| 299 | + |
| 300 | + |
| 301 | +def get_func(func, aliasing, implementations): |
| 302 | + """Return the key of a found implementation or the func itself""" |
| 303 | + try: |
| 304 | + func_str = aliasing[func] |
| 305 | + except KeyError: |
| 306 | + if callable(func): |
| 307 | + return func |
| 308 | + else: |
| 309 | + if func_str in implementations: |
| 310 | + return func_str |
| 311 | + if func_str.startswith("nan") and func_str[3:] in funcs_no_separate_nan: |
| 312 | + raise ValueError(f"{func_str[3:]} does not have a nan-version") |
| 313 | + else: |
| 314 | + raise NotImplementedError("No such function available") |
| 315 | + raise ValueError(f"func {func} is neither a valid function string nor a callable object") |
| 316 | + |
| 317 | + |
| 318 | +def build_dispatch(impl_dict, aliasing): |
| 319 | + """Fuse an aliasing and an implementation table for single-lookup dispatch. |
| 320 | +
|
| 321 | + Returns a dict mapping every alias (string or callable) onto a |
| 322 | + ``(canonical name, implementation)`` tuple. Aliases without an |
| 323 | + implementation are omitted, so lookups missing there fall through to |
| 324 | + ``get_func``, which keeps handling custom callables and error reporting. |
| 325 | + """ |
| 326 | + dispatch = {} |
| 327 | + for alias, name in aliasing.items(): |
| 328 | + impl = impl_dict.get(name) |
| 329 | + if impl is not None: |
| 330 | + dispatch[alias] = (name, impl) |
| 331 | + return dispatch |
| 332 | + |
| 333 | + |
| 334 | +_forced_float_types = { |
| 335 | + "mean", |
| 336 | + "median", |
| 337 | + "trapezoid", |
| 338 | + "nanmean", |
| 339 | + "nanmedian", |
| 340 | + "nantrapezoid", |
| 341 | +} |
| 342 | + |
| 343 | +# var/std (and sumofsquares) measure the squared magnitude of the deviations, |
| 344 | +# which is real even for complex input - like np.var returning a real dtype |
| 345 | +_forced_real_float_types = { |
| 346 | + "var", |
| 347 | + "std", |
| 348 | + "nanvar", |
| 349 | + "nanstd", |
| 350 | + "sumofsquares", |
| 351 | + "nansumofsquares", |
| 352 | +} |
| 353 | + |
| 354 | +# functions whose result is not complex even for complex input - only these may |
| 355 | +# be asked for a non-complex dtype, everything else would throw the imaginary |
| 356 | +# part of the values away |
| 357 | +# the names of the dtype table in utils (which holds the numpy dtypes |
| 358 | +# themselves) - everything here may aggregate complex input into a |
| 359 | +# non-complex dtype without throwing the imaginary part away |
| 360 | +_complex_real_result = ( |
| 361 | + frozenset( |
| 362 | + [ |
| 363 | + "array", |
| 364 | + "all", |
| 365 | + "any", |
| 366 | + "nanall", |
| 367 | + "nanany", |
| 368 | + "len", |
| 369 | + "nanlen", |
| 370 | + "allnan", |
| 371 | + "anynan", |
| 372 | + "argmax", |
| 373 | + "argmin", |
| 374 | + "nanargmin", |
| 375 | + "nanargmax", |
| 376 | + ] |
| 377 | + ) |
| 378 | + | _forced_real_float_types |
| 379 | +) |
| 380 | + |
| 381 | + |
| 382 | +def check_complex_dtype(a_dtype, dtype, func_str): |
| 383 | + """Complain when a non-complex ``dtype`` is requested for complex input. |
| 384 | +
|
| 385 | + ``resolve_output_dtype`` applies this for the backends that honour ``dtype``; |
| 386 | + the pure python implementation needs it on its own, since python numbers |
| 387 | + carry no dtype and it would otherwise silently ignore the request. |
| 388 | + """ |
| 389 | + if ( |
| 390 | + dtype is not None |
| 391 | + and _is_complex_dtype(a_dtype) |
| 392 | + and not _is_complex_dtype(dtype) |
| 393 | + and func_str not in _complex_real_result |
| 394 | + ): |
| 395 | + raise TypeError( |
| 396 | + f"function {func_str} cannot aggregate complex values into the non-complex dtype " |
| 397 | + f"{dtype} - pass a complex dtype to keep the imaginary part" |
| 398 | + ) |
0 commit comments