Merge pull request #227 from kori73/join-example

add join example to dataframe tutorial

Author: quasiben

.gitignore

@@ -10,6 +10,7 @@ data/random.hdf5
 data/weather-big
 data/myfile.hdf5
 data/flightjson
+data/holidays
 data/nycflights
 data/myfile.zarr
 data/accounts.parquet

04_dataframe.ipynb

@@ -241,6 +241,33 @@
     "df.tail()  # now works"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Let's also read the holidays data which will use in the exercises"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "holidays = dd.read_parquet(os.path.join('data', \"holidays\"))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": [
+    "holidays.head()"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},
@@ -460,6 +487,35 @@
     "df.groupby(\"DayOfWeek\").DepDelay.mean().compute()"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### 6.) What holiday has the worst average departure delay?\n",
+    "\n",
+    "*Hint*: use [`df.merge`](https://pandas.pydata.org/pandas-docs/stable/user_guide/merging.html) to bring holiday information.\n",
+    "\n",
+    "*Note*: If you have prepared the dataset with `--small` argument or set the `DASK_TUTORIAL_SMALL` environment variable to `True`, you might see only a couple of holidays. This is because the small dataset contains a limited number of rows."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Your code here"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df.merge(holidays, on=[\"Date\"], how=\"left\").groupby(\"holiday\").DepDelay.mean().compute()"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},
@@ -817,7 +873,7 @@
  "metadata": {
   "anaconda-cloud": {},
   "kernelspec": {
-   "display_name": "Python 3",
+   "display_name": "Python 3 (ipykernel)",
    "language": "python",
    "name": "python3"
   },
@@ -831,7 +887,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.7.6"
+   "version": "3.8.12"
   }
  },
  "nbformat": 4,

binder/environment.yml

@@ -7,6 +7,7 @@ dependencies:
   - jupyterlab>=2.0.0,<3
   - numpy>=1.18.1
   - h5py
+  - holidays
   - scipy>=1.3.0
   - toolz
   - bokeh>=2.0.0

prep.py

@@ -11,6 +11,7 @@
 import h5py
 import numpy as np
 import pandas as pd
+import holidays
 from skimage.transform import resize
 
 from accounts import account_entries, account_params, json_entries
@@ -40,6 +41,28 @@ def parse_args(args=None):
                   'directory.')
 
 
+def holiday():
+    holidays_dir = os.path.join(data_dir, "holidays")
+    if os.path.exists(holidays_dir):
+        return
+
+    years = [
+        1990, 1991, 1992, 1993, 1994,
+        1995, 1996, 1997, 1998, 1999
+    ]
+    holidays_dict = holidays.US(years=years)
+    us_holidays = pd.DataFrame(
+        data={
+            "Date": holidays_dict.keys(),
+            "holiday": holidays_dict.values()
+        },
+    )
+    us_holidays = us_holidays.assign(
+        Date=us_holidays.Date.astype("datetime64[ns]"))
+    us_holidays.to_parquet(holidays_dir)
+    print("Created holidays data.")
+
+
 def flights(small=None):
     start = time.time()
     flights_raw = os.path.join(data_dir, 'nycflights.tar.gz')
@@ -224,6 +247,7 @@ def main(args=None):
         accounts_json(args.small)
     if args.dataset == "flights" or args.dataset == "all":
         flights(args.small)
+        holiday()
 
 
 if __name__ == '__main__':