Merge pull request #173 from mmccarty/data-sources

Configurable data source locations

Author: martindurant

01x_lazy.ipynb

@@ -412,6 +412,17 @@
     "Similarly, we can launch Python processes and threads in the background. Some methods allow mapping over multiple inputs and gathering the results, more on that later.  The thread starts and the cell completes immediately, but the data associated with the download only appears in the queue object some time later."
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Edit sources.py to configure source locations\n",
+    "import sources\n",
+    "sources.lazy_url"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -428,7 +439,7 @@
     "    q.put(u.read())\n",
     "\n",
     "q = queue.Queue()\n",
-    "t = threading.Thread(target=get_webdata, args=('http://www.google.com', q))\n",
+    "t = threading.Thread(target=get_webdata, args=(sources.lazy_url, q))\n",
     "t.start()"
    ]
   },

02_bag.ipynb

@@ -120,6 +120,17 @@
     "b.take(1)"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Edit sources.py to configure source locations\n",
+    "import sources\n",
+    "sources.bag_url"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -128,7 +139,7 @@
    "source": [
     "# Requires `s3fs` library\n",
     "# each partition is a remote CSV text file\n",
-    "b = db.read_text('s3://dask-data/nyc-taxi/2015/yellow_tripdata_2015-01.csv',\n",
+    "b = db.read_text(sources.bag_url,\n",
     "                 storage_options={'anon': True})\n",
     "b.take(1)"
    ]

prep.py

@@ -15,6 +15,8 @@
 
 from accounts import account_entries, account_params, json_entries
 
+import sources
+
 DATASETS = ["random", "weather", "accounts", "flights", "all"]
 here = os.path.dirname(__file__)
 data_dir = os.path.abspath(os.path.join(here, 'data'))
@@ -53,7 +55,7 @@ def flights(small=None):
 
     if not os.path.exists(flights_raw):
         print("- Downloading NYC Flights dataset... ", end='', flush=True)
-        url = "https://storage.googleapis.com/dask-tutorial-data/nycflights.tar.gz"
+        url = sources.flights_url
         urllib.request.urlretrieve(url, flights_raw)
         print("done", flush=True)
 

sources.py

@@ -0,0 +1,3 @@
+flights_url = "https://storage.googleapis.com/dask-tutorial-data/nycflights.tar.gz"
+lazy_url = "http://www.google.com"
+bag_url = "s3://dask-data/nyc-taxi/2015/yellow_tripdata_2015-01.csv"