extend processing (#17)

Author: jmoralez

.github/workflows/ci.yaml

@@ -16,13 +16,13 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  test:
+  all-tests:
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
         os: [macos-latest, ubuntu-latest, windows-latest]
-        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11']
+        python-version: ['3.8', '3.9', '3.10', '3.11']
     steps:
       - name: Clone repo
         uses: actions/checkout@v3
@@ -38,5 +38,26 @@ jobs:
         run: pip install ./
 
       - name: Run tests
-        run: nbdev_test --do_print --timing
+        run: nbdev_test --do_print --timing --flags 'matplotlib polars'
 
+  minimal-tests:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [macos-latest, ubuntu-latest, windows-latest]
+    steps:
+      - name: Clone repo
+        uses: actions/checkout@v3
+
+      - name: Set up python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+          cache: 'pip'
+
+      - name: Install dependencies
+        run: pip3 install . nbdev
+
+      - name: Run tests
+        run: nbdev_test --do_print --timing --skip_file_re '(plotting|target_transforms)'

nbs/compat.ipynb

@@ -32,23 +32,15 @@
    "source": [
     "#| export\n",
     "try:\n",
+    "    import polars as pl\n",
     "    from polars import DataFrame as pl_DataFrame\n",
     "    from polars import Series as pl_Series\n",
-    "    from polars import col as pl_col\n",
-    "    from polars import concat as pl_concat\n",
-    "    from polars import lit as pl_lit\n",
     "\n",
     "    POLARS_INSTALLED = True\n",
     "except ImportError:\n",
-    "    class pl_DataFrame:  # type: ignore\n",
-    "        ...\n",
-    "\n",
-    "    class pl_Series:  # type: ignore\n",
-    "        ...\n",
-    "\n",
-    "    pl_col = None  # type: ignore\n",
-    "    pl_concat = None  # type: ignore\n",
-    "    pl_lit = None  # type: ignore\n",
+    "    pl = None\n",
+    "    pl_DataFrame = type(None)\n",
+    "    pl_Series = type(None)\n",
     "\n",
     "    POLARS_INSTALLED = False\n",
     "\n",

nbs/data.ipynb

@@ -55,7 +55,7 @@
     "import numpy as np\n",
     "import pandas as pd\n",
     "\n",
-    "from utilsforecast.compat import DataFrame"
+    "from utilsforecast.compat import DataFrame, pl"
    ]
   },
   {
@@ -171,8 +171,6 @@
     "            df[cat_cols] = df[cat_cols].astype('category')\n",
     "            df['unique_id'] = df['unique_id'].cat.as_ordered()\n",
     "    else:\n",
-    "        import polars as pl\n",
-    "\n",
     "        df = pl.DataFrame(vals_dict)\n",
     "        df = df.with_columns(pl.col('unique_id').sort())\n",
     "        if static_as_categorical:\n",

nbs/evaluation.ipynb

@@ -46,7 +46,7 @@
     "\n",
     "import pandas as pd\n",
     "\n",
-    "from utilsforecast.compat import DataFrame, pl_concat, pl_lit"
+    "from utilsforecast.compat import DataFrame, pl"
    ]
   },
   {
@@ -152,14 +152,14 @@
     "        if isinstance(result, pd.DataFrame):\n",
     "            result['metric'] = metric_name\n",
     "        else:\n",
-    "            result = result.with_columns(pl_lit(metric_name).alias('metric'))\n",
+    "            result = result.with_columns(pl.lit(metric_name).alias('metric'))\n",
     "        results_per_metric.append(result)\n",
     "    if isinstance(df, pd.DataFrame):\n",
     "        df = pd.concat(results_per_metric).reset_index(drop=True)\n",
     "        out_cols = [c for c in df.columns if c not in (id_col, 'metric')]\n",
     "        df = df[[id_col, 'metric', *out_cols]]\n",
     "    else:\n",
-    "        df = pl_concat(results_per_metric, how='diagonal')\n",
+    "        df = pl.concat(results_per_metric, how='diagonal')\n",
     "        out_cols = [c for c in df.columns if c not in (id_col, 'metric')]\n",
     "        df = df.select([id_col, 'metric', *out_cols])\n",
     "    return df"
@@ -259,7 +259,6 @@
     "\n",
     "import numpy as np\n",
     "import pandas as pd\n",
-    "import polars as pl\n",
     "\n",
     "from utilsforecast.losses import *\n",
     "from utilsforecast.data import generate_series"
@@ -272,8 +271,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "series = generate_series(10, n_models=2, level=[80])\n",
-    "series_pl = generate_series(10, n_models=2, level=[80], engine='polars')"
+    "series = generate_series(10, n_models=2, level=[80])"
    ]
   },
   {
@@ -654,6 +652,8 @@
    "outputs": [],
    "source": [
     "#| hide\n",
+    "#| polars\n",
+    "series_pl = generate_series(10, n_models=2, level=[80], engine='polars')\n",
     "pl_evaluation = (\n",
     "    evaluate(\n",
     "        series_pl,\n",

nbs/grouped_array.ipynb

@@ -24,7 +24,7 @@
     "import pandas as pd\n",
     "\n",
     "from utilsforecast.compat import DataFrame\n",
-    "from utilsforecast.processing import DataFrameProcessor"
+    "from utilsforecast.processing import DataFrameProcessor, group_by"
    ]
   },
   {
@@ -168,15 +168,12 @@
     "        if isinstance(df, pd.DataFrame):\n",
     "            sizes = df.groupby(id_col, observed=True).size().values\n",
     "        else:\n",
-    "            try:\n",
-    "                group_sizes = df.group_by(id_col, maintain_order=True).count()\n",
-    "            except AttributeError:\n",
-    "                group_sizes = df.groupby(id_col, maintain_order=True).count()\n",
+    "            group_sizes = group_by(df, id_col, maintain_order=True).count()\n",
     "            sizes = group_sizes['count'].to_numpy()\n",
     "        \n",
     "        indptr = np.append(0, sizes.cumsum())\n",
     "        proc = DataFrameProcessor(id_col, time_col, target_col)\n",
-    "        data = proc._value_cols_to_numpy(df)\n",
+    "        data = proc.value_cols_to_numpy(df)\n",
     "        if data.dtype not in (np.float32, np.float64):\n",
     "            data = data.astype(np.float32)\n",
     "        return cls(data, indptr)\n",
@@ -232,7 +229,9 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from fastcore.test import test_eq, test_fail"
+    "from fastcore.test import test_eq, test_fail\n",
+    "\n",
+    "from utilsforecast.data import generate_series"
    ]
   },
   {
@@ -334,24 +333,15 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "8c28ccf5-0f75-4cae-a0d5-e880037ff3e1",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from utilsforecast.data import generate_series"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "6914dcf3-4d13-42f7-917c-242a47477740",
+   "id": "c06325e0-8265-4a61-b936-7fc29d6396be",
    "metadata": {},
    "outputs": [],
    "source": [
+    "#| polars\n",
     "# build from df\n",
     "series_pd = generate_series(10, static_as_categorical=False, engine='pandas')\n",
-    "series_pl = generate_series(10, static_as_categorical=False, engine='polars')\n",
     "ga_pd = GroupedArray.from_sorted_df(series_pd, 'unique_id', 'ds', 'y')\n",
+    "series_pl = generate_series(10, static_as_categorical=False, engine='polars')\n",
     "ga_pl = GroupedArray.from_sorted_df(series_pl, 'unique_id', 'ds', 'y')\n",
     "np.testing.assert_allclose(ga_pd.data, ga_pl.data)\n",
     "np.testing.assert_equal(ga_pd.indptr, ga_pl.indptr)"

nbs/imgs/index.png

@@ -207,6 +207,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "#| matplotlib\n",
     "from utilsforecast.plotting import plot_series"
    ]
   },
@@ -216,6 +217,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "#| matplotlib\n",
     "fig = plot_series(series, plot_random=False, max_insample_length=50, engine='matplotlib')\n",
     "fig.savefig('imgs/index.png', bbox_inches='tight')"
    ]