Merge pull request #18 from catalyst-cooperative/dev

Transition to using hand-crafted versioned data URLs

Author: zaneselvans

README.rst

@@ -110,10 +110,6 @@ this is weird).
 
 .. code:: py
 
-   # Environment variables that tell Intake where to find and cache data
-   os.environ["PUDL_INTAKE_CACHE"] = str(Path.home() / ".cache/intake")
-   os.environ["PUDL_INTAKE_PATH"] = "gs://intake.catalyst.coop/test"
-
    import intake
    import pandas as pd
    from pudl_catalog.helpers import year_state_filter
@@ -143,7 +139,7 @@ but with all the Jinja template fields interpolated and filled in:
        storage_options:
          simplecache:
            cache_storage: /home/zane/.cache/intake
-       urlpath: simplecache::gs://intake.catalyst.coop/test/hourly_emissions_epacems.parquet
+       urlpath: simplecache::gs://intake.catalyst.coop/dev/hourly_emissions_epacems.parquet
      description: Hourly pollution emissions and plant operational data reported via
        Continuous Emissions Monitoring Systems (CEMS) as required by 40 CFR Part 75.
        Includes CO2, NOx, and SO2, as well as the heat content of fuel consumed and gross

notebooks/pudl-catalog.ipynb

@@ -9,13 +9,6 @@
     "# Notebook Preamble"
    ]
   },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## IPython Magic"
-   ]
-  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -26,15 +19,6 @@
     "%autoreload 3"
    ]
   },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "tags": []
-   },
-   "source": [
-    "## Notebook Imports"
-   ]
-  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -47,13 +31,19 @@
     "import sys\n",
     "from pathlib import Path\n",
     "\n",
-    "# We need to set these environment variables prior to importing our intake catalog.\n",
-    "# You can also set them in your own shell environment instead.\n",
-    "os.environ[\"PUDL_INTAKE_CACHE\"] = str(Path.home() / \".cache/intake\")\n",
-    "os.environ[\"PUDL_INTAKE_PATH\"] = \"gs://intake.catalyst.coop/test\"\n",
+    "logger = logging.getLogger()\n",
+    "logger.setLevel(logging.INFO)\n",
+    "handler = logging.StreamHandler(stream=sys.stdout)\n",
+    "formatter = logging.Formatter(\"%(message)s\")\n",
+    "handler.setFormatter(formatter)\n",
+    "logger.handlers = [handler]\n",
+    "\n",
+    "# Where to cache downloaded data locally. Defaults to ~/.intake/cache\n",
+    "# os.environ[\"PUDL_INTAKE_CACHE\"] = str(Path.home() / \".cache/intake\")\n",
     "\n",
-    "# Local data if you've got it!\n",
+    "# You can override the default path to the data in your environment if need be\n",
     "# os.environ[\"PUDL_INTAKE_PATH\"] = str(Path.cwd().parent() / \"data\")\n",
+    "# os.environ[\"PUDL_INTAKE_PATH\"] = \"gs://intake.catalyst.coop/dev\"\n",
     "\n",
     "# 3rd Party Imports:\n",
     "import intake\n",
@@ -68,28 +58,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "## Set up a logger"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "logger = logging.getLogger()\n",
-    "logger.setLevel(logging.INFO)\n",
-    "handler = logging.StreamHandler(stream=sys.stdout)\n",
-    "formatter = logging.Formatter(\"%(message)s\")\n",
-    "handler.setFormatter(formatter)\n",
-    "logger.handlers = [handler]"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## What Intake data sources are installed?"
+    "# Explore installed Intake catalogs"
    ]
   },
   {
@@ -155,7 +124,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "## Normal usage"
+    "# Reading some data from the catalog"
    ]
   },
   {
@@ -212,7 +181,7 @@
    "outputs": [],
    "source": [
     "%%time\n",
-    "df1 = pd.read_parquet(\"gs://intake.catalyst.coop/test/hourly_emissions_epacems/epacems-2020-ID.parquet\")"
+    "df1 = pd.read_parquet(f\"{os.environ['PUDL_INTAKE_PATH']}/hourly_emissions_epacems/epacems-2020-ID.parquet\")"
    ]
   },
   {
@@ -251,7 +220,7 @@
     "from pprint import pprint\n",
     "import fsspec\n",
     "epacems_pq = pq.read_table(\n",
-    "    \"gs://intake.catalyst.coop/test/hourly_emissions_epacems/epacems-2020-ID.parquet\",\n",
+    "    f\"{os.environ['PUDL_INTAKE_PATH']}/hourly_emissions_epacems/epacems-2020-ID.parquet\",\n",
     "    filesystem=fsspec.filesystem(\"gs\"),\n",
     ")\n",
     "dtype_dict = {name: dtype for name, dtype in zip(epacems_pq.schema.names, epacems_pq.schema.types)}\n",

setup.py

@@ -20,6 +20,7 @@
     url="https://github.com/catalyst-cooperative/pudl-catalog",
     project_urls={
         "Source": "https://github.com/catalyst-cooperative/pudl-catalog",
+        "Documentation": "https://catalystcoop-pudl-catalog.readthedocs.io",
         "Issue Tracker": "https://github.com/catalyst-cooperative/pudl-catalog/issues",
     },
     classifiers=[

src/pudl_catalog/__init__.py

@@ -12,9 +12,9 @@
 logger.addHandler(logging.NullHandler())
 
 BASE_URLS = {
-    "gs": "gs://intake.catalyst.coop/test",
+    "gs": "gs://intake.catalyst.coop/v0.1.0",
     # HTTPS access doesn't really work well, so we're hiding it from users for now.
-    "https": "https://storage.googleapis.com/intake.catalyst.coop/test",
+    "https": "https://storage.googleapis.com/intake.catalyst.coop/v0.1.0",
 }
 
 # Ensure that the user has set the relevant environment variables
@@ -23,6 +23,7 @@
         "Environment variable PUDL_INTAKE_PATH is not set. "
         f"Defaulting to {BASE_URLS['gs']}"
     )
+    os.environ["PUDL_INTAKE_PATH"] = BASE_URLS["gs"]
 
 if os.getenv("PUDL_INTAKE_CACHE") is None:
     logger.info(