Merge pull request #380 from hydrosquall/feature/Issue-376-update-enigma-connector

Feature/Issue 376 Update EnigmaReader Data Connector

Author: davidastephens

docs/source/remote_data.rst

@@ -204,15 +204,20 @@ Available expiry dates can be accessed from the ``expiry_dates`` property.
 Enigma
 ======
 
-Access datasets from `Enigma <https://app.enigma.io>`__,
-the world's largest repository of structured public data.
+Access datasets from `Enigma <https://public.enigma.com>`__,
+the world's largest repository of structured public data. Note that the Enigma
+URL has changed from `app.enigma.io <https://app.enigma.io>`__ as of release 
+``0.6.0``, as the old API deprecated.
+
+Datasets are unique identified by the ``uuid4`` at the end of a dataset's web address.
+For example, the following code downloads from  `USDA Food Recalls 1996 Data <https://public.enigma.com/datasets/292129b0-1275-44c8-a6a3-2a0881f24fe1>`__.
 
 .. ipython:: python
 
     import os
     import pandas_datareader as pdr
 
-    df = pdr.get_data_enigma('enigma.trade.ams.toxic.2015', os.getenv('ENIGMA_API_KEY'))
+    df = pdr.get_data_enigma('292129b0-1275-44c8-a6a3-2a0881f24fe1', os.getenv('ENIGMA_API_KEY'))
     df.columns
 
 .. _remote_data.quandl:

pandas_datareader/data.py

@@ -137,7 +137,7 @@ def DataReader(name, data_source=None, start=None, end=None,
                                  session=session).read()
 
     elif data_source == "enigma":
-        return EnigmaReader(datapath=name, api_key=access_key).read()
+        return EnigmaReader(dataset_id=name, api_key=access_key).read()
 
     elif data_source == "fred":
         return FredReader(symbols=name, start=start, end=end,

pandas_datareader/enigma.py

@@ -1,43 +1,44 @@
-import zlib
 import os
 import time
-from pandas.compat import StringIO
 
+from pandas.compat import StringIO
 import pandas.compat as compat
 import pandas as pd
-import requests
 
 from pandas_datareader.base import _BaseReader
 
 
 class EnigmaReader(_BaseReader):
     """
-    Collects Enigma data located at the specified datapath and
-    returns a pandas DataFrame.
+    Collects current snapshot of Enigma data located at the specified
+    dataset ID and returns a pandas DataFrame.
+
+    # Example
+    Download current snapshot for the following Florida Inspections Dataset
+    https://public.enigma.com/datasets/bedaf052-5fcd-4758-8d27-048ce8746c6a
 
     Usage (high-level):
     ```
         import pandas_datareader as pdr
-        df = pdr.get_data_enigma('enigma.inspections.restaurants.fl')
+        df = pdr.get_data_enigma('bedaf052-5fcd-4758-8d27-048ce8746c6a')
 
         # in the event that ENIGMA_API_KEY does not exist in your env,
         # it can be supplied as the second arg:
-        df = prd.get_data_enigma('enigma.inspections.restaurants.fl',
-        ...                      'ARIAMFHKJMISF38UT')
+        df = prd.get_data_enigma('bedaf052-5fcd-4758-8d27-048ce8746c6a',
+        ...                      'INSERT_API_KEY')
     ```
 
     Usage:
     ```
-        df = EnigmaReader(datapath='enigma.inspections.restaurants.fl',
-        ...               api_key='ARIAMFHKJMISF38UT').read()
+        df = EnigmaReader(dataset_id='bedaf052-5fcd-4758-8d27-048ce8746c6a',
+        ...               api_key='INSERT_API_KEY').read()
     ```
     """
-
     def __init__(self,
-                 datapath=None,
+                 dataset_id=None,
                  api_key=None,
                  retry_count=5,
-                 pause=0.250,
+                 pause=.75,
                  session=None):
 
         super(EnigmaReader, self).__init__(symbols=[],
@@ -49,64 +50,69 @@ def __init__(self,
                 raise ValueError("Please provide an Enigma API key or set "
                                  "the ENIGMA_API_KEY environment variable\n"
                                  "If you do not have an API key, you can get "
-                                 "one here: https://app.enigma.io/signup")
+                                 "one here: http://public.enigma.com/signup")
         else:
             self._api_key = api_key
 
-        self._datapath = datapath
-        if not isinstance(self._datapath, compat.string_types):
+        self._dataset_id = dataset_id
+        if not isinstance(self._dataset_id, compat.string_types):
             raise ValueError(
-                "The Enigma datapath must be a string (ex: "
-                "'enigma.inspections.restaurants.fl')")
-
-    @property
-    def url(self):
-        return 'https://api.enigma.io/v2/export/{}/{}'.format(self._api_key,
-                                                              self._datapath)
-
-    @property
-    def export_key(self):
-        return 'export_url'
+                "The Enigma dataset_id must be a string (ex: "
+                "'bedaf052-5fcd-4758-8d27-048ce8746c6a')")
+
+        headers = {
+            'Authorization': 'Bearer {0}'.format(self._api_key),
+            'User-Agent': 'pandas-datareader',
+        }
+        self.session.headers.update(headers)
+        self._base_url = "https://public.enigma.com/api"
+        self._retry_count = retry_count
+        self._retry_delay = pause
 
-    @property
-    def _head_key(self):
-        return 'head_url'
-
-    def _request(self, url):
-        self.session.headers.update({'User-Agent': 'pandas-datareader'})
-        resp = self.session.get(url)
-        resp.raise_for_status()
-        return resp
-
-    def _decompress_export(self, compressed_export_data):
-        return zlib.decompress(compressed_export_data, 16 + zlib.MAX_WBITS)
+    def read(self):
+        try:
+            return self._read()
+        finally:
+            self.close()
 
-    def extract_export_url(self, delay=10, max_attempts=10):
-        """
-        Performs an HTTP HEAD request on 'head_url' until it returns a `200`.
-        This allows the Enigma API time to export the requested data.
-        """
-        resp = self._request(self.url)
+    def _read(self):
+        snapshot_id = self.get_current_snapshot_id(self._dataset_id)
+        exported_data = self.get_snapshot_export(snapshot_id)  # TODO: Retry?
+        decoded_data = exported_data.decode("utf-8")
+        return pd.read_csv(StringIO(decoded_data))
+
+    def _get(self, url):
+        """HTTP GET Request with Retry Logic"""
+        url = "{0}/{1}".format(self._base_url, url)
         attempts = 0
         while True:
             try:
-                requests.head(resp.json()[self._head_key]).raise_for_status()
+                response = self.session.get(url)
+                response.raise_for_status()
+                return response
             except Exception as e:
-                attempts += 1
-                if attempts > max_attempts:
+                if attempts < self._retry_count:
+                    attempts += 1
+                    time.sleep(self._retry_delay)
+                    continue
+                else:
                     raise e
-                time.sleep(delay)
-                continue
-            return resp.json()[self.export_key]
 
-    def read(self):
-        try:
-            return self._read()
-        finally:
-            self.close()
+    def get_current_snapshot_id(self, dataset_id):
+        """Get ID of the most current snapshot of a dataset"""
+        dataset_metadata = self.get_dataset_metadata(dataset_id)
+        return dataset_metadata['current_snapshot']['id']
 
-    def _read(self):
-        export_gzipped_req = self._request(self.extract_export_url())
-        decompressed_data = self._decompress_export(
-            export_gzipped_req.content).decode("utf-8")
-        return pd.read_csv(StringIO(decompressed_data))
+    def get_dataset_metadata(self, dataset_id):
+        """Get the Dataset Model of this EnigmaReader's dataset
+            https://docs.public.enigma.com/resources/dataset/index.html
+        """
+        url = "datasets/{0}?row_limit=0".format(dataset_id)
+        response = self._get(url)
+        return response.json()
+
+    def get_snapshot_export(self, snapshot_id):
+        """Return raw CSV of a dataset"""
+        url = "export/{0}".format(snapshot_id)
+        response = self._get(url)
+        return response.content

pandas_datareader/tests/test_enigma.py

@@ -11,32 +11,40 @@
 @pytest.mark.skipif(TEST_API_KEY is None, reason="no enigma_api_key")
 class TestEnigma(object):
 
+    @property
+    def dataset_id(self):
+        """
+        USDA Food Recall Archive - 1996
+        Selected for being a relatively small dataset.
+        https://public.enigma.com/datasets/292129b0-1275-44c8-a6a3-2a0881f24fe1
+        """
+        return "292129b0-1275-44c8-a6a3-2a0881f24fe1"
+
     @classmethod
     def setup_class(cls):
         pytest.importorskip("lxml")
 
     def test_enigma_datareader(self):
         try:
-            df = web.DataReader('enigma.inspections.restaurants.fl',
+            df = web.DataReader(self.dataset_id,
                                 'enigma', access_key=TEST_API_KEY)
-            assert 'serialid' in df.columns
+            assert 'case_number' in df.columns
         except HTTPError as e:
             pytest.skip(e)
 
     def test_enigma_get_data_enigma(self):
         try:
-            df = pdr.get_data_enigma(
-                'enigma.inspections.restaurants.fl', TEST_API_KEY)
-            assert 'serialid' in df.columns
+            df = pdr.get_data_enigma(self.dataset_id, TEST_API_KEY)
+            assert 'case_number' in df.columns
         except HTTPError as e:
             pytest.skip(e)
 
     def test_bad_key(self):
         with pytest.raises(HTTPError):
-            web.DataReader('enigma.inspections.restaurants.fl',
+            web.DataReader(self.dataset_id,
                            'enigma', access_key=TEST_API_KEY + 'xxx')
 
-    def test_bad_url(self):
+    def test_bad_dataset_id(self):
         with pytest.raises(HTTPError):
-            web.DataReader('enigma.inspections.restaurants.fllzzy',
+            web.DataReader('zzzzzzzz-zzzz-zzzz-zzzz-zzzzzzzzzzz',
                            'enigma', access_key=TEST_API_KEY)