{ "cells": [ { "cell_type": "markdown", "id": "84bf8d8e-0536-4673-82be-55b6897627d6", "metadata": {}, "source": [ "# Dask support\n", "Like xarray, xarray-einstats also aims to support using their functions on\n", "NumPy or Dask backed xarray objects.\n", "\n", "Functions in xarray-einstats should support Dask backed xarray objects.\n", "But Dask support is still not extensively tested.\n", "If you have issues using Dask backed xarray objects please open an [issue](https://github.com/arviz-devs/xarray-einstats)\n", "\n", "Dask support within xarray-einstats requires explicit activation by users, exactly like it happens with {func}`xarray.apply_ufunc`.\n", "In fact, what is happening is xarray-einstats forwards any unrecognized keyword argument (e.g. `dask` or `dask_gufunc_kwargs`) to `apply_ufunc`." ] }, { "cell_type": "code", "execution_count": 1, "id": "b373d879-c129-4441-9186-d4f98ddcb699", "metadata": {}, "outputs": [], "source": [ "import dask.array as da\n", "import xarray as xr\n", "from xarray_einstats import linalg\n", "\n", "xr.set_options(display_expand_data=False);" ] }, { "cell_type": "code", "execution_count": 2, "id": "e3daba12-78f5-473f-b9fc-71665434ec34", "metadata": {}, "outputs": [], "source": [ "matrix_batch = xr.DataArray(da.random.normal(size=(1000, 77, 77), chunks=\"auto\"), dims=[\"batch\", \"dim1\", \"dim2\"])" ] }, { "cell_type": "markdown", "id": "724165af-0437-423f-8c22-3c0f5980eb52", "metadata": {}, "source": [ "Trying to use xarray-einstats functions directly on Dask arrays triggers an error:" ] }, { "cell_type": "code", "execution_count": 3, "id": "02bdc253-ff87-4481-979a-1db478f093d4", "metadata": { "deletable": true, "editable": true, "slideshow": { "slide_type": "" }, "tags": [ "hide-output" ] }, "outputs": [ { "ename": "ValueError", "evalue": "apply_ufunc encountered a chunked array on an argument, but handling for chunked arrays has not been enabled. Either set the ``dask`` argument or load your data into memory first with ``.load()`` or ``.compute()``", "output_type": "error", "traceback": [ "\u001b[31m---------------------------------------------------------------------------\u001b[39m", "\u001b[31mValueError\u001b[39m Traceback (most recent call last)", "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[3]\u001b[39m\u001b[32m, line 1\u001b[39m\n\u001b[32m----> \u001b[39m\u001b[32m1\u001b[39m linalg.norm(matrix_batch, dims=[\u001b[33m\"dim1\"\u001b[39m, \u001b[33m\"dim2\"\u001b[39m])\n", "\u001b[36mFile \u001b[39m\u001b[32m~/Documents/repos_oss/xarray-einstats/src/xarray_einstats/linalg.py:801\u001b[39m, in \u001b[36mnorm\u001b[39m\u001b[34m(da, dims, ord, **kwargs)\u001b[39m\n\u001b[32m 799\u001b[39m in_dims = dims\n\u001b[32m 800\u001b[39m norm_kwargs[\u001b[33m\"\u001b[39m\u001b[33maxis\u001b[39m\u001b[33m\"\u001b[39m] = (-\u001b[32m2\u001b[39m, -\u001b[32m1\u001b[39m)\n\u001b[32m--> \u001b[39m\u001b[32m801\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[30;43mxr\u001b[39;49m\u001b[30;43m.\u001b[39;49m\u001b[30;43mapply_ufunc\u001b[39;49m\u001b[30;43m(\u001b[39;49m\n\u001b[32m 802\u001b[39m \u001b[30;43m \u001b[39;49m\u001b[30;43mnp\u001b[39;49m\u001b[30;43m.\u001b[39;49m\u001b[30;43mlinalg\u001b[39;49m\u001b[30;43m.\u001b[39;49m\u001b[30;43mnorm\u001b[39;49m\u001b[30;43m,\u001b[39;49m\u001b[30;43m \u001b[39;49m\u001b[30;43mda\u001b[39;49m\u001b[30;43m,\u001b[39;49m\u001b[30;43m \u001b[39;49m\u001b[30;43minput_core_dims\u001b[39;49m\u001b[30;43m=\u001b[39;49m\u001b[30;43m[\u001b[39;49m\u001b[30;43min_dims\u001b[39;49m\u001b[30;43m]\u001b[39;49m\u001b[30;43m,\u001b[39;49m\u001b[30;43m \u001b[39;49m\u001b[30;43mkwargs\u001b[39;49m\u001b[30;43m=\u001b[39;49m\u001b[30;43mnorm_kwargs\u001b[39;49m\u001b[30;43m,\u001b[39;49m\u001b[30;43m \u001b[39;49m\u001b[30;43m*\u001b[39;49m\u001b[30;43m*\u001b[39;49m\u001b[30;43mkwargs\u001b[39;49m\n\u001b[32m 803\u001b[39m \u001b[30;43m\u001b[39;49m\u001b[30;43m)\u001b[39;49m\n", "\u001b[36mFile \u001b[39m\u001b[32m~/bin/miniforge3/envs/general/lib/python3.14/site-packages/xarray/computation/apply_ufunc.py:1267\u001b[39m, in \u001b[36mapply_ufunc\u001b[39m\u001b[34m(func, input_core_dims, output_core_dims, exclude_dims, vectorize, join, dataset_join, dataset_fill_value, keep_attrs, kwargs, dask, output_dtypes, output_sizes, meta, dask_gufunc_kwargs, on_missing_core_dim, *args)\u001b[39m\n\u001b[32m 1265\u001b[39m \u001b[38;5;66;03m# feed DataArray apply_variable_ufunc through apply_dataarray_vfunc\u001b[39;00m\n\u001b[32m 1266\u001b[39m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;28many\u001b[39m(\u001b[38;5;28misinstance\u001b[39m(a, DataArray) \u001b[38;5;28;01mfor\u001b[39;00m a \u001b[38;5;129;01min\u001b[39;00m args):\n\u001b[32m-> \u001b[39m\u001b[32m1267\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[30;43mapply_dataarray_vfunc\u001b[39;49m\u001b[30;43m(\u001b[39;49m\n\u001b[32m 1268\u001b[39m \u001b[30;43m \u001b[39;49m\u001b[30;43mvariables_vfunc\u001b[39;49m\u001b[30;43m,\u001b[39;49m\n\u001b[32m 1269\u001b[39m \u001b[30;43m \u001b[39;49m\u001b[30;43m*\u001b[39;49m\u001b[30;43margs\u001b[39;49m\u001b[30;43m,\u001b[39;49m\n\u001b[32m 1270\u001b[39m \u001b[30;43m \u001b[39;49m\u001b[30;43msignature\u001b[39;49m\u001b[30;43m=\u001b[39;49m\u001b[30;43msignature\u001b[39;49m\u001b[30;43m,\u001b[39;49m\n\u001b[32m 1271\u001b[39m \u001b[30;43m \u001b[39;49m\u001b[30;43mjoin\u001b[39;49m\u001b[30;43m=\u001b[39;49m\u001b[30;43mjoin\u001b[39;49m\u001b[30;43m,\u001b[39;49m\n\u001b[32m 1272\u001b[39m \u001b[30;43m \u001b[39;49m\u001b[30;43mexclude_dims\u001b[39;49m\u001b[30;43m=\u001b[39;49m\u001b[30;43mexclude_dims\u001b[39;49m\u001b[30;43m,\u001b[39;49m\n\u001b[32m 1273\u001b[39m \u001b[30;43m \u001b[39;49m\u001b[30;43mkeep_attrs\u001b[39;49m\u001b[30;43m=\u001b[39;49m\u001b[30;43mkeep_attrs\u001b[39;49m\u001b[30;43m,\u001b[39;49m\n\u001b[32m 1274\u001b[39m \u001b[30;43m \u001b[39;49m\u001b[30;43m)\u001b[39;49m\n\u001b[32m 1275\u001b[39m \u001b[38;5;66;03m# feed Variables directly through apply_variable_ufunc\u001b[39;00m\n\u001b[32m 1276\u001b[39m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;28many\u001b[39m(\u001b[38;5;28misinstance\u001b[39m(a, Variable) \u001b[38;5;28;01mfor\u001b[39;00m a \u001b[38;5;129;01min\u001b[39;00m args):\n", "\u001b[36mFile \u001b[39m\u001b[32m~/bin/miniforge3/envs/general/lib/python3.14/site-packages/xarray/computation/apply_ufunc.py:312\u001b[39m, in \u001b[36mapply_dataarray_vfunc\u001b[39m\u001b[34m(func, signature, join, exclude_dims, keep_attrs, *args)\u001b[39m\n\u001b[32m 307\u001b[39m result_coords, result_indexes = build_output_coords_and_indexes(\n\u001b[32m 308\u001b[39m args, signature, exclude_dims, combine_attrs=keep_attrs\n\u001b[32m 309\u001b[39m )\n\u001b[32m 311\u001b[39m data_vars = [\u001b[38;5;28mgetattr\u001b[39m(a, \u001b[33m\"\u001b[39m\u001b[33mvariable\u001b[39m\u001b[33m\"\u001b[39m, a) \u001b[38;5;28;01mfor\u001b[39;00m a \u001b[38;5;129;01min\u001b[39;00m args]\n\u001b[32m--> \u001b[39m\u001b[32m312\u001b[39m result_var = \u001b[30;43mfunc\u001b[39;49m\u001b[30;43m(\u001b[39;49m\u001b[30;43m*\u001b[39;49m\u001b[30;43mdata_vars\u001b[39;49m\u001b[30;43m)\u001b[39;49m\n\u001b[32m 314\u001b[39m out: \u001b[38;5;28mtuple\u001b[39m[DataArray, ...] | DataArray\n\u001b[32m 315\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m signature.num_outputs > \u001b[32m1\u001b[39m:\n", "\u001b[36mFile \u001b[39m\u001b[32m~/bin/miniforge3/envs/general/lib/python3.14/site-packages/xarray/computation/apply_ufunc.py:741\u001b[39m, in \u001b[36mapply_variable_ufunc\u001b[39m\u001b[34m(func, signature, exclude_dims, dask, output_dtypes, vectorize, keep_attrs, dask_gufunc_kwargs, *args)\u001b[39m\n\u001b[32m 739\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28many\u001b[39m(is_chunked_array(array) \u001b[38;5;28;01mfor\u001b[39;00m array \u001b[38;5;129;01min\u001b[39;00m input_data):\n\u001b[32m 740\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m dask == \u001b[33m\"\u001b[39m\u001b[33mforbidden\u001b[39m\u001b[33m\"\u001b[39m:\n\u001b[32m--> \u001b[39m\u001b[32m741\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[32m 742\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mapply_ufunc encountered a chunked array on an \u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m 743\u001b[39m \u001b[33m\"\u001b[39m\u001b[33margument, but handling for chunked arrays has not \u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m 744\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mbeen enabled. Either set the ``dask`` argument \u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m 745\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mor load your data into memory first with \u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m 746\u001b[39m \u001b[33m\"\u001b[39m\u001b[33m``.load()`` or ``.compute()``\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m 747\u001b[39m )\n\u001b[32m 748\u001b[39m \u001b[38;5;28;01melif\u001b[39;00m dask == \u001b[33m\"\u001b[39m\u001b[33mparallelized\u001b[39m\u001b[33m\"\u001b[39m:\n\u001b[32m 749\u001b[39m chunkmanager = get_chunked_array_type(*input_data)\n", "\u001b[31mValueError\u001b[39m: apply_ufunc encountered a chunked array on an argument, but handling for chunked arrays has not been enabled. Either set the ``dask`` argument or load your data into memory first with ``.load()`` or ``.compute()``" ] } ], "source": [ "linalg.norm(matrix_batch, dims=[\"dim1\", \"dim2\"])" ] }, { "cell_type": "markdown", "id": "298e23b9-d271-4e68-a2c2-02dd661eb8bc", "metadata": { "deletable": true, "editable": true, "slideshow": { "slide_type": "" }, "tags": [] }, "source": [ "We need to allow dask inputs through `dask=\"allowed\"` or `dask=\"parallelized\"`. In general, `allowed` is preferred, but it only works if there is a Dask implementation of the requested function available, `parallelized` works in most cases but is usually slower and still not a silver bullet. Consequently, xarray-einstats does not set the `dask` argument automatically when the input is a dask array, and finding which of the two will work and be better might require a bit of trial an error an also be dependent on the Dask version available.\n", "\n", "Here is one example with `linalg.norm` for which both alternatives work:" ] }, { "cell_type": "code", "execution_count": 4, "id": "b03c96a5-23b7-4fde-8ee9-b56911559a19", "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", " \n", " \n", " \n", " \n", "
\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
Array Chunk
Bytes 7.81 kiB 7.81 kiB
Shape (1000,) (1000,)
Dask graph 1 chunks in 6 graph layers
Data type float64 numpy.ndarray
\n", "
\n", " \n", "\n", " \n", " \n", " \n", "\n", " \n", " \n", " \n", "\n", " \n", " \n", "\n", " \n", " 1000\n", " 1\n", "\n", "
" ], "text/plain": [ "dask.array" ] }, "execution_count": 4, "metadata": {}, "output_type": "execute_result" } ], "source": [ "norms = linalg.norm(matrix_batch, dims=[\"dim1\", \"dim2\"], dask=\"allowed\")\n", "norms.data" ] }, { "cell_type": "code", "execution_count": 5, "id": "3287e425-a7da-4756-87d9-1b9b43e7bad5", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "
<xarray.DataArray 'normal-758a57bb75ed3127c94212c3e83a5ed8' (batch: 1000)> Size: 8kB\n",
       "77.07 77.61 76.86 77.53 77.99 77.22 76.9 ... 77.92 78.03 77.96 76.6 77.31 76.26\n",
       "Dimensions without coordinates: batch
" ], "text/plain": [ " Size: 8kB\n", "77.07 77.61 76.86 77.53 77.99 77.22 76.9 ... 77.92 78.03 77.96 76.6 77.31 76.26\n", "Dimensions without coordinates: batch" ] }, "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ "norms.compute()" ] }, { "cell_type": "code", "execution_count": 6, "id": "b19b98df-3256-443f-b6e4-6501d76b6bc1", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "
<xarray.DataArray 'normal-758a57bb75ed3127c94212c3e83a5ed8' (batch: 1000)> Size: 8kB\n",
       "77.07 77.61 76.86 77.53 77.99 77.22 76.9 ... 77.92 78.03 77.96 76.6 77.31 76.26\n",
       "Dimensions without coordinates: batch
" ], "text/plain": [ " Size: 8kB\n", "77.07 77.61 76.86 77.53 77.99 77.22 76.9 ... 77.92 78.03 77.96 76.6 77.31 76.26\n", "Dimensions without coordinates: batch" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "linalg.norm(matrix_batch, dims=[\"dim1\", \"dim2\"], dask=\"parallelized\").compute()" ] }, { "cell_type": "markdown", "id": "0d03e25c-c592-414e-869f-efd2bf033dc1", "metadata": {}, "source": [ ":::{seealso}\n", "{ref}`Xarray documentation `\n", "\n", "[Dask examples](https://examples.dask.org/xarray.html)\n", ":::" ] }, { "cell_type": "code", "execution_count": 7, "id": "c336f026-9460-4ee8-bbd4-d3343ae4ddcb", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Last updated: Mon, 20 Jul 2026\n", "\n", "Python implementation: CPython\n", "Python version : 3.14.6\n", "IPython version : 9.15.0\n", "\n", "dask : 2026.7.1\n", "xarray : 2026.7.0\n", "xarray_einstats: 0.11.0\n", "\n", "Watermark: 2.6.0\n", "\n" ] } ], "source": [ "%load_ext watermark\n", "%watermark -n -u -v -iv -w" ] }, { "cell_type": "code", "execution_count": null, "id": "eb4ffe5e-7f62-4523-8974-d5db0f26aaa0", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.14.6" } }, "nbformat": 4, "nbformat_minor": 5 }