[tune] Parallel Coordinate Visualization Notebook (#1218)

Author: richardliaw

python/ray/tune/ParallelCoordinatesVisualization.ipynb

@@ -0,0 +1,128 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Tune Visualization"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "In order to visualize results, please install `plotly` with the following command:\n",
+    "\n",
+    "  `pip install plotly`"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "from ray.tune.visual_utils import load_results_to_df, generate_plotly_dim_dict\n",
+    "import plotly\n",
+    "import plotly.graph_objs as go\n",
+    "plotly.offline.init_notebook_mode(connected=True)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Specify the directory where all your results are in the variable `RESULTS_DIR`."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "RESULTS_DIR = \"/tmp/ray/\"\n",
+    "df = load_results_to_df(RESULTS_DIR)\n",
+    "[key for key in df]"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Choose the fields you wish to visualize over in `GOOD_FIELDS`."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true,
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": [
+    "GOOD_FIELDS = ['experiment_id',\n",
+    "               'num_sgd_iter',\n",
+    "               'timesteps_total',\n",
+    "               'episode_len_mean',\n",
+    "               'episode_reward_mean']\n",
+    "\n",
+    "visualization_df = df[GOOD_FIELDS]\n",
+    "visualization_df"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Enjoy.\n",
+    "\n",
+    "Documentation for this Plotly visualization can be found here: https://plot.ly/python/parallel-coordinates-plot/"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "data = [go.Parcoords(\n",
+    "            line = dict(color = 'blue'),\n",
+    "            dimensions = [generate_plotly_dim_dict(visualization_df, field) \n",
+    "                              for field in visualization_df])\n",
+    "]\n",
+    "\n",
+    "plotly.offline.iplot(data)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.6.2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

python/ray/tune/visual_utils.py

@@ -0,0 +1,82 @@
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import pandas as pd
+from pandas.api.types import is_string_dtype, is_numeric_dtype
+
+import os
+import os.path as osp
+import numpy as np
+import json
+
+
+def _flatten_dict(dt):
+    while any(type(v) is dict for v in dt.values()):
+        remove = []
+        add = {}
+        for key, value in dt.items():
+            if type(value) is dict:
+                for subkey, v in value.items():
+                    add[":".join([key, subkey])] = v
+                remove.append(key)
+        dt.update(add)
+        for k in remove:
+            del dt[k]
+    return dt
+
+
+def _parse_results(res_path):
+    res_dict = {}
+    try:
+        with open(res_path) as f:
+            # Get last line in file
+            for line in f:
+                pass
+        res_dict = _flatten_dict(json.loads(line.strip()))
+    except Exception as e:
+        print("Importing %s failed...Perhaps empty?" % res_path)
+    return res_dict
+
+
+def _parse_configs(cfg_path):
+    try:
+        with open(cfg_path) as f:
+            cfg_dict = _flatten_dict(json.load(f))
+    except Exception as e:
+        print(e)
+    return cfg_dict
+
+
+def _resolve(directory, result_fname):
+    resultp = osp.join(directory, result_fname)
+    res_dict = _parse_results(resultp)
+    cfgp = osp.join(directory, "config.json")
+    cfg_dict = _parse_configs(cfgp)
+    cfg_dict.update(res_dict)
+    return cfg_dict
+
+
+def load_results_to_df(directory, result_name="result.json"):
+    exp_directories = [dirpath for dirpath, dirs, files in os.walk(directory)
+                       for f in files if f == result_name]
+    data = [_resolve(directory, result_name) for directory in exp_directories]
+    return pd.DataFrame(data)
+
+
+def generate_plotly_dim_dict(df, field):
+    dim_dict = {}
+    dim_dict["label"] = field
+    column = df[field]
+    if is_numeric_dtype(column):
+        dim_dict["values"] = column
+    elif is_string_dtype(column):
+        texts = column.unique()
+        dim_dict["values"] = [np.argwhere(texts == x).flatten()[0]
+                              for x in column]
+        dim_dict["tickvals"] = list(range(len(texts)))
+        dim_dict["ticktext"] = texts
+    else:
+        raise Exception("Unidentifiable Type")
+
+    return dim_dict